Mercurial > hg
changeset 30542:64b55bffc1c0
merge with stable
| author | Augie Fackler <augie@google.com> |
|---|---|
| date | Sun, 27 Nov 2016 20:44:52 -0500 |
| parents | 342d0cb4f446 (diff) 819f96b82fa4 (current diff) |
| children | 03fae9048fa1 |
| files | hgext/shelve.py i18n/ja.po i18n/pt_BR.po tests/test-shelve.t |
| diffstat | 247 files changed, 29199 insertions(+), 3646 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Mon Nov 14 22:43:25 2016 +0100 +++ b/.hgignore Sun Nov 27 20:44:52 2016 -0500 @@ -49,6 +49,7 @@ tags cscope.* .idea/* +.asv/* i18n/hg.pot locale/*/LC_MESSAGES/hg.mo hgext/__index__.py
--- a/Makefile Mon Nov 14 22:43:25 2016 +0100 +++ b/Makefile Sun Nov 27 20:44:52 2016 -0500 @@ -207,6 +207,12 @@ docker-ubuntu-xenial-ppa: contrib/docker/ubuntu-xenial contrib/dockerdeb ubuntu xenial --source-only +docker-ubuntu-yakkety: contrib/docker/ubuntu-yakkety + contrib/dockerdeb ubuntu yakkety + +docker-ubuntu-yakkety-ppa: contrib/docker/ubuntu-yakkety + contrib/dockerdeb ubuntu yakkety --source-only + fedora20: mkdir -p packages/fedora20 contrib/buildrpm
--- a/contrib/all-revsets.txt Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/all-revsets.txt Sun Nov 27 20:44:52 2016 -0500 @@ -8,7 +8,7 @@ # # Please update this file with any revsets you use for benchmarking a change so # that future contributors can easily find and retest it when doing further -# modification. Feel free to highlight interresting variants if needed. +# modification. Feel free to highlight interesting variants if needed. ## Revset from this section are all extracted from changelog when this file was
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/asv.conf.json Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,13 @@ +{ + "version": 1, + "project": "mercurial", + "project_url": "https://mercurial-scm.org/", + "repo": "..", + "branches": ["default", "stable"], + "environment_type": "virtualenv", + "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/", + "benchmark_dir": "benchmarks", + "env_dir": "../.asv/env", + "results_dir": "../.asv/results", + "html_dir": "../.asv/html" +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/benchmarks/__init__.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,102 @@ +# __init__.py - asv benchmark suite +# +# Copyright 2016 Logilab SA <contact@logilab.fr> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +'''ASV (https://asv.readthedocs.io) benchmark suite + +Benchmark are parameterized against reference repositories found in the +directory pointed by the REPOS_DIR environment variable. + +Invocation example: + + $ export REPOS_DIR=~/hgperf/repos + # run suite on given revision + $ asv --config contrib/asv.conf.json run REV + # run suite on new changesets found in stable and default branch + $ asv --config contrib/asv.conf.json run NEW + # display a comparative result table of benchmark results between two given + # revisions + $ asv --config contrib/asv.conf.json compare REV1 REV2 + # compute regression detection and generate ASV static website + $ asv --config contrib/asv.conf.json publish + # serve the static website + $ asv --config contrib/asv.conf.json preview +''' + +from __future__ import absolute_import + +import functools +import os +import re + +from mercurial import ( + extensions, + hg, + ui as uimod, +) + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.path.pardir, os.path.pardir)) +reposdir = os.environ['REPOS_DIR'] +reposnames = [name for name in os.listdir(reposdir) + if os.path.isdir(os.path.join(reposdir, name, ".hg"))] +if not reposnames: + raise ValueError("No repositories found in $REPO_DIR") +outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys ' + r'\d+.\d+ \(best of \d+\)')) + +def runperfcommand(reponame, command, *args, **kwargs): + os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "") + ui = uimod.ui() + repo = hg.repository(ui, os.path.join(reposdir, reponame)) + perfext = extensions.load(ui, 'perfext', + os.path.join(basedir, 'contrib', 'perf.py')) + cmd = getattr(perfext, command) + ui.pushbuffer() + cmd(ui, repo, *args, **kwargs) + output = ui.popbuffer() + match = outputre.search(output) + if not match: + raise ValueError("Invalid output {0}".format(output)) + return float(match.group(1)) + +def perfbench(repos=reposnames, name=None, params=None): + """decorator to declare ASV benchmark based on contrib/perf.py extension + + An ASV benchmark is a python function with the given attributes: + + __name__: should start with track_, time_ or mem_ to be collected by ASV + params and param_name: parameter matrix to display multiple graphs on the + same page. + pretty_name: If defined it's displayed in web-ui instead of __name__ + (useful for revsets) + the module name is prepended to the benchmark name and displayed as + "category" in webui. + + Benchmarks are automatically parameterized with repositories found in the + REPOS_DIR environment variable. + + `params` is the param matrix in the form of a list of tuple + (param_name, [value0, value1]) + + For example [(x, [a, b]), (y, [c, d])] declare benchmarks for + (a, c), (a, d), (b, c) and (b, d). + """ + params = list(params or []) + params.insert(0, ("repo", repos)) + + def decorator(func): + @functools.wraps(func) + def wrapped(repo, *args): + def perf(command, *a, **kw): + return runperfcommand(repo, command, *a, **kw) + return func(perf, *args) + + wrapped.params = [p[1] for p in params] + wrapped.param_names = [p[0] for p in params] + wrapped.pretty_name = name + return wrapped + return decorator
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/benchmarks/perf.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,26 @@ +# perf.py - asv benchmarks using contrib/perf.py extension +# +# Copyright 2016 Logilab SA <contact@logilab.fr> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from . import perfbench + +@perfbench() +def track_tags(perf): + return perf("perftags") + +@perfbench() +def track_status(perf): + return perf("perfstatus", unknown=False) + +@perfbench(params=[('rev', ['1000', '10000', 'tip'])]) +def track_manifest(perf, rev): + return perf("perfmanifest", rev) + +@perfbench() +def track_heads(perf): + return perf("perfheads")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/benchmarks/revset.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,53 @@ +# revset.py - asv revset benchmarks +# +# Copyright 2016 Logilab SA <contact@logilab.fr> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +'''ASV revset benchmarks generated from contrib/base-revsets.txt + +Each revset benchmark is parameterized with variants (first, last, sort, ...) +''' + +from __future__ import absolute_import + +import os +import string +import sys + +from . import basedir, perfbench + +def createrevsetbenchmark(baseset, variants=None): + if variants is None: + # Default variants + variants = ["plain", "first", "last", "sort", "sort+first", + "sort+last"] + fname = "track_" + "_".join("".join([ + c if c in string.digits + string.letters else " " + for c in baseset + ]).split()) + + def wrap(fname, baseset): + @perfbench(name=baseset, params=[("variant", variants)]) + def f(perf, variant): + revset = baseset + if variant != "plain": + for var in variant.split("+"): + revset = "%s(%s)" % (var, revset) + return perf("perfrevset", revset) + f.__name__ = fname + return f + return wrap(fname, baseset) + +def initializerevsetbenchmarks(): + mod = sys.modules[__name__] + with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'), + 'rb') as fh: + for line in fh: + baseset = line.strip() + if baseset and not baseset.startswith('#'): + func = createrevsetbenchmark(baseset) + setattr(mod, func.__name__, func) + +initializerevsetbenchmarks()
--- a/contrib/chg/Makefile Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/chg/Makefile Sun Nov 27 20:44:52 2016 -0500 @@ -40,7 +40,6 @@ [ -d $(CHGSOCKDIR) ] || ( umask 077; mkdir $(CHGSOCKDIR) ) $(HG) serve --cwd / --cmdserver chgunix \ --address $(CHGSOCKNAME) \ - --config extensions.chgserver= \ --config cmdserver.log=/dev/stderr .PHONY: clean
--- a/contrib/chg/chg.c Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/chg/chg.c Sun Nov 27 20:44:52 2016 -0500 @@ -225,7 +225,6 @@ "--cmdserver", "chgunix", "--address", opts->sockname, "--daemon-postexec", "chdir:/", - "--config", "extensions.chgserver=", }; size_t baseargvsize = sizeof(baseargv) / sizeof(baseargv[0]); size_t argsize = baseargvsize + opts->argsize + 1;
--- a/contrib/debugshell.py Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/debugshell.py Sun Nov 27 20:44:52 2016 -0500 @@ -18,7 +18,7 @@ 'mercurial': mercurial, 'repo': repo, 'cl': repo.changelog, - 'mf': repo.manifest, + 'mf': repo.manifestlog, } code.interact(msg, local=objects) @@ -27,7 +27,7 @@ import IPython cl = repo.changelog - mf = repo.manifest + mf = repo.manifestlog cl, mf # use variables to appease pyflakes IPython.embed()
--- a/contrib/memory.py Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/memory.py Sun Nov 27 20:44:52 2016 -0500 @@ -25,8 +25,8 @@ key = parts[0][2:-1].lower() if key in result: result[key] = int(parts[1]) - ui.write_err(", ".join(["%s: %.1f MiB" % (key, value / 1024.0) - for key, value in result.iteritems()]) + "\n") + ui.write_err(", ".join(["%s: %.1f MiB" % (k, v / 1024.0) + for k, v in result.iteritems()]) + "\n") def extsetup(ui): atexit.register(memusage, ui)
--- a/contrib/perf.py Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/perf.py Sun Nov 27 20:44:52 2016 -0500 @@ -25,6 +25,7 @@ import sys import time from mercurial import ( + bdiff, changegroup, cmdutil, commands, @@ -135,13 +136,14 @@ if opts is None: opts = {} - # redirect all to stderr - ui = ui.copy() - uifout = safeattrsetter(ui, 'fout', ignoremissing=True) - if uifout: - # for "historical portability": - # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d) - uifout.set(ui.ferr) + # redirect all to stderr unless buffer api is in use + if not ui._buffers: + ui = ui.copy() + uifout = safeattrsetter(ui, 'fout', ignoremissing=True) + if uifout: + # for "historical portability": + # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d) + uifout.set(ui.ferr) # get a formatter uiformatter = getattr(ui, 'formatter', None) @@ -565,8 +567,8 @@ ctx = scmutil.revsingle(repo, rev, rev) t = ctx.manifestnode() def d(): - repo.manifest.clearcaches() - repo.manifest.read(t) + repo.manifestlog.clearcaches() + repo.manifestlog[t].read() timer(d) fm.end() @@ -746,6 +748,64 @@ timer(d) fm.end() +@command('perfbdiff', revlogopts + formatteropts + [ + ('', 'count', 1, 'number of revisions to test (when using --startrev)'), + ('', 'alldata', False, 'test bdiffs for all associated revisions')], + '-c|-m|FILE REV') +def perfbdiff(ui, repo, file_, rev=None, count=None, **opts): + """benchmark a bdiff between revisions + + By default, benchmark a bdiff between its delta parent and itself. + + With ``--count``, benchmark bdiffs between delta parents and self for N + revisions starting at the specified revision. + + With ``--alldata``, assume the requested revision is a changeset and + measure bdiffs for all changes related to that changeset (manifest + and filelogs). + """ + if opts['alldata']: + opts['changelog'] = True + + if opts.get('changelog') or opts.get('manifest'): + file_, rev = None, file_ + elif rev is None: + raise error.CommandError('perfbdiff', 'invalid arguments') + + textpairs = [] + + r = cmdutil.openrevlog(repo, 'perfbdiff', file_, opts) + + startrev = r.rev(r.lookup(rev)) + for rev in range(startrev, min(startrev + count, len(r) - 1)): + if opts['alldata']: + # Load revisions associated with changeset. + ctx = repo[rev] + mtext = repo.manifestlog._revlog.revision(ctx.manifestnode()) + for pctx in ctx.parents(): + pman = repo.manifestlog._revlog.revision(pctx.manifestnode()) + textpairs.append((pman, mtext)) + + # Load filelog revisions by iterating manifest delta. + man = ctx.manifest() + pman = ctx.p1().manifest() + for filename, change in pman.diff(man).items(): + fctx = repo.file(filename) + f1 = fctx.revision(change[0][0] or -1) + f2 = fctx.revision(change[1][0] or -1) + textpairs.append((f1, f2)) + else: + dp = r.deltaparent(rev) + textpairs.append((r.revision(dp), r.revision(rev))) + + def d(): + for pair in textpairs: + bdiff.bdiff(*pair) + + timer, fm = gettimer(ui, opts) + timer(d) + fm.end() + @command('perfdiffwd', formatteropts) def perfdiffwd(ui, repo, **opts): """Profile diff of working directory changes""" @@ -799,6 +859,84 @@ timer(d) fm.end() +@command('perfrevlogchunks', revlogopts + formatteropts + + [('s', 'startrev', 0, 'revision to start at')], + '-c|-m|FILE') +def perfrevlogchunks(ui, repo, file_=None, startrev=0, **opts): + """Benchmark operations on revlog chunks. + + Logically, each revlog is a collection of fulltext revisions. However, + stored within each revlog are "chunks" of possibly compressed data. This + data needs to be read and decompressed or compressed and written. + + This command measures the time it takes to read+decompress and recompress + chunks in a revlog. It effectively isolates I/O and compression performance. + For measurements of higher-level operations like resolving revisions, + see ``perfrevlog`` and ``perfrevlogrevision``. + """ + rl = cmdutil.openrevlog(repo, 'perfrevlogchunks', file_, opts) + revs = list(rl.revs(startrev, len(rl) - 1)) + + def rlfh(rl): + if rl._inline: + return getsvfs(repo)(rl.indexfile) + else: + return getsvfs(repo)(rl.datafile) + + def doread(): + rl.clearcaches() + for rev in revs: + rl._chunkraw(rev, rev) + + def doreadcachedfh(): + rl.clearcaches() + fh = rlfh(rl) + for rev in revs: + rl._chunkraw(rev, rev, df=fh) + + def doreadbatch(): + rl.clearcaches() + rl._chunkraw(revs[0], revs[-1]) + + def doreadbatchcachedfh(): + rl.clearcaches() + fh = rlfh(rl) + rl._chunkraw(revs[0], revs[-1], df=fh) + + def dochunk(): + rl.clearcaches() + fh = rlfh(rl) + for rev in revs: + rl._chunk(rev, df=fh) + + chunks = [None] + + def dochunkbatch(): + rl.clearcaches() + fh = rlfh(rl) + # Save chunks as a side-effect. + chunks[0] = rl._chunks(revs, df=fh) + + def docompress(): + rl.clearcaches() + for chunk in chunks[0]: + rl.compress(chunk) + + benches = [ + (lambda: doread(), 'read'), + (lambda: doreadcachedfh(), 'read w/ reused fd'), + (lambda: doreadbatch(), 'read batch'), + (lambda: doreadbatchcachedfh(), 'read batch w/ reused fd'), + (lambda: dochunk(), 'chunk'), + (lambda: dochunkbatch(), 'chunk batch'), + (lambda: docompress(), 'compress'), + ] + + for fn, title in benches: + timer, fm = gettimer(ui, opts) + timer(fn, title=title) + fm.end() + @command('perfrevlogrevision', revlogopts + formatteropts + [('', 'cache', False, 'use caches instead of clearing')], '-c|-m|FILE REV')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/LICENSE Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,27 @@ +Copyright (c) 2016, Gregory Szorc +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/MANIFEST.in Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,2 @@ +graft zstd +include make_cffi.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/NEWS.rst Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,63 @@ +Version History +=============== + +0.5.0 (released 2016-11-10) +--------------------------- + +* Vendored version of zstd updated to 1.1.1. +* Continuous integration for Python 3.6 and 3.7 +* Continuous integration for Conda +* Added compression and decompression APIs providing similar interfaces + to the standard library ``zlib`` and ``bz2`` modules. This allows + coding to a common interface. +* ``zstd.__version__` is now defined. +* ``read_from()`` on various APIs now accepts objects implementing the buffer + protocol. +* ``read_from()`` has gained a ``skip_bytes`` argument. This allows callers + to pass in an existing buffer with a header without having to create a + slice or a new object. +* Implemented ``ZstdCompressionDict.as_bytes()``. +* Python's memory allocator is now used instead of ``malloc()``. +* Low-level zstd data structures are reused in more instances, cutting down + on overhead for certain operations. +* ``distutils`` boilerplate for obtaining an ``Extension`` instance + has now been refactored into a standalone ``setup_zstd.py`` file. This + allows other projects with ``setup.py`` files to reuse the + ``distutils`` code for this project without copying code. +* The monolithic ``zstd.c`` file has been split into a header file defining + types and separate ``.c`` source files for the implementation. + +History of the Project +====================== + +2016-08-31 - Zstandard 1.0.0 is released and Gregory starts hacking on a +Python extension for use by the Mercurial project. A very hacky prototype +is sent to the mercurial-devel list for RFC. + +2016-09-03 - Most functionality from Zstandard C API implemented. Source +code published on https://github.com/indygreg/python-zstandard. Travis-CI +automation configured. 0.0.1 release on PyPI. + +2016-09-05 - After the API was rounded out a bit and support for Python +2.6 and 2.7 was added, version 0.1 was released to PyPI. + +2016-09-05 - After the compressor and decompressor APIs were changed, 0.2 +was released to PyPI. + +2016-09-10 - 0.3 is released with a bunch of new features. ZstdCompressor +now accepts arguments controlling frame parameters. The source size can now +be declared when performing streaming compression. ZstdDecompressor.decompress() +is implemented. Compression dictionaries are now cached when using the simple +compression and decompression APIs. Memory size APIs added. +ZstdCompressor.read_from() and ZstdDecompressor.read_from() have been +implemented. This rounds out the major compression/decompression APIs planned +by the author. + +2016-10-02 - 0.3.3 is released with a bug fix for read_from not fully +decoding a zstd frame (issue #2). + +2016-10-02 - 0.4.0 is released with zstd 1.1.0, support for custom read and +write buffer sizes, and a few bug fixes involving failure to read/write +all data when buffer sizes were too small to hold remaining data. + +2016-11-10 - 0.5.0 is released with zstd 1.1.1 and other enhancements.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/README.rst Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,776 @@ +================ +python-zstandard +================ + +This project provides a Python C extension for interfacing with the +`Zstandard <http://www.zstd.net>`_ compression library. + +The primary goal of the extension is to provide a Pythonic interface to +the underlying C API. This means exposing most of the features and flexibility +of the C API while not sacrificing usability or safety that Python provides. + +| |ci-status| |win-ci-status| + +State of Project +================ + +The project is officially in beta state. The author is reasonably satisfied +with the current API and that functionality works as advertised. There +may be some backwards incompatible changes before 1.0. Though the author +does not intend to make any major changes to the Python API. + +There is continuous integration for Python versions 2.6, 2.7, and 3.3+ +on Linux x86_x64 and Windows x86 and x86_64. The author is reasonably +confident the extension is stable and works as advertised on these +platforms. + +Expected Changes +---------------- + +The author is reasonably confident in the current state of what's +implemented on the ``ZstdCompressor`` and ``ZstdDecompressor`` types. +Those APIs likely won't change significantly. Some low-level behavior +(such as naming and types expected by arguments) may change. + +There will likely be arguments added to control the input and output +buffer sizes (currently, certain operations read and write in chunk +sizes using zstd's preferred defaults). + +There should be an API that accepts an object that conforms to the buffer +interface and returns an iterator over compressed or decompressed output. + +The author is on the fence as to whether to support the extremely +low level compression and decompression APIs. It could be useful to +support compression without the framing headers. But the author doesn't +believe it a high priority at this time. + +The CFFI bindings are half-baked and need to be finished. + +Requirements +============ + +This extension is designed to run with Python 2.6, 2.7, 3.3, 3.4, and 3.5 +on common platforms (Linux, Windows, and OS X). Only x86_64 is currently +well-tested as an architecture. + +Installing +========== + +This package is uploaded to PyPI at https://pypi.python.org/pypi/zstandard. +So, to install this package:: + + $ pip install zstandard + +Binary wheels are made available for some platforms. If you need to +install from a source distribution, all you should need is a working C +compiler and the Python development headers/libraries. On many Linux +distributions, you can install a ``python-dev`` or ``python-devel`` +package to provide these dependencies. + +Packages are also uploaded to Anaconda Cloud at +https://anaconda.org/indygreg/zstandard. See that URL for how to install +this package with ``conda``. + +Performance +=========== + +Very crude and non-scientific benchmarking (most benchmarks fall in this +category because proper benchmarking is hard) show that the Python bindings +perform within 10% of the native C implementation. + +The following table compares the performance of compressing and decompressing +a 1.1 GB tar file comprised of the files in a Firefox source checkout. Values +obtained with the ``zstd`` program are on the left. The remaining columns detail +performance of various compression APIs in the Python bindings. + ++-------+-----------------+-----------------+-----------------+---------------+ +| Level | Native | Simple | Stream In | Stream Out | +| | Comp / Decomp | Comp / Decomp | Comp / Decomp | Comp | ++=======+=================+=================+=================+===============+ +| 1 | 490 / 1338 MB/s | 458 / 1266 MB/s | 407 / 1156 MB/s | 405 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 2 | 412 / 1288 MB/s | 381 / 1203 MB/s | 345 / 1128 MB/s | 349 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 3 | 342 / 1312 MB/s | 319 / 1182 MB/s | 285 / 1165 MB/s | 287 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 11 | 64 / 1506 MB/s | 66 / 1436 MB/s | 56 / 1342 MB/s | 57 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ + +Again, these are very unscientific. But it shows that Python is capable of +compressing at several hundred MB/s and decompressing at over 1 GB/s. + +Comparison to Other Python Bindings +=================================== + +https://pypi.python.org/pypi/zstd is an alternative Python binding to +Zstandard. At the time this was written, the latest release of that +package (1.0.0.2) had the following significant differences from this package: + +* It only exposes the simple API for compression and decompression operations. + This extension exposes the streaming API, dictionary training, and more. +* It adds a custom framing header to compressed data and there is no way to + disable it. This means that data produced with that module cannot be used by + other Zstandard implementations. + +Bundling of Zstandard Source Code +================================= + +The source repository for this project contains a vendored copy of the +Zstandard source code. This is done for a few reasons. + +First, Zstandard is relatively new and not yet widely available as a system +package. Providing a copy of the source code enables the Python C extension +to be compiled without requiring the user to obtain the Zstandard source code +separately. + +Second, Zstandard has both a stable *public* API and an *experimental* API. +The *experimental* API is actually quite useful (contains functionality for +training dictionaries for example), so it is something we wish to expose to +Python. However, the *experimental* API is only available via static linking. +Furthermore, the *experimental* API can change at any time. So, control over +the exact version of the Zstandard library linked against is important to +ensure known behavior. + +Instructions for Building and Testing +===================================== + +Once you have the source code, the extension can be built via setup.py:: + + $ python setup.py build_ext + +We recommend testing with ``nose``:: + + $ nosetests + +A Tox configuration is present to test against multiple Python versions:: + + $ tox + +Tests use the ``hypothesis`` Python package to perform fuzzing. If you +don't have it, those tests won't run. + +There is also an experimental CFFI module. You need the ``cffi`` Python +package installed to build and test that. + +To create a virtualenv with all development dependencies, do something +like the following:: + + # Python 2 + $ virtualenv venv + + # Python 3 + $ python3 -m venv venv + + $ source venv/bin/activate + $ pip install cffi hypothesis nose tox + +API +=== + +The compiled C extension provides a ``zstd`` Python module. This module +exposes the following interfaces. + +ZstdCompressor +-------------- + +The ``ZstdCompressor`` class provides an interface for performing +compression operations. + +Each instance is associated with parameters that control compression +behavior. These come from the following named arguments (all optional): + +level + Integer compression level. Valid values are between 1 and 22. +dict_data + Compression dictionary to use. + + Note: When using dictionary data and ``compress()`` is called multiple + times, the ``CompressionParameters`` derived from an integer compression + ``level`` and the first compressed data's size will be reused for all + subsequent operations. This may not be desirable if source data size + varies significantly. +compression_params + A ``CompressionParameters`` instance (overrides the ``level`` value). +write_checksum + Whether a 4 byte checksum should be written with the compressed data. + Defaults to False. If True, the decompressor can verify that decompressed + data matches the original input data. +write_content_size + Whether the size of the uncompressed data will be written into the + header of compressed data. Defaults to False. The data will only be + written if the compressor knows the size of the input data. This is + likely not true for streaming compression. +write_dict_id + Whether to write the dictionary ID into the compressed data. + Defaults to True. The dictionary ID is only written if a dictionary + is being used. + +Simple API +^^^^^^^^^^ + +``compress(data)`` compresses and returns data as a one-shot operation.:: + + cctx = zstd.ZsdCompressor() + compressed = cctx.compress(b'data to compress') + +Streaming Input API +^^^^^^^^^^^^^^^^^^^ + +``write_to(fh)`` (which behaves as a context manager) allows you to *stream* +data into a compressor.:: + + cctx = zstd.ZstdCompressor(level=10) + with cctx.write_to(fh) as compressor: + compressor.write(b'chunk 0') + compressor.write(b'chunk 1') + ... + +The argument to ``write_to()`` must have a ``write(data)`` method. As +compressed data is available, ``write()`` will be called with the comrpessed +data as its argument. Many common Python types implement ``write()``, including +open file handles and ``io.BytesIO``. + +``write_to()`` returns an object representing a streaming compressor instance. +It **must** be used as a context manager. That object's ``write(data)`` method +is used to feed data into the compressor. + +If the size of the data being fed to this streaming compressor is known, +you can declare it before compression begins:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh, size=data_len) as compressor: + compressor.write(chunk0) + compressor.write(chunk1) + ... + +Declaring the size of the source data allows compression parameters to +be tuned. And if ``write_content_size`` is used, it also results in the +content size being written into the frame header of the output data. + +The size of chunks being ``write()`` to the destination can be specified:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh, write_size=32768) as compressor: + ... + +To see how much memory is being used by the streaming compressor:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh) as compressor: + ... + byte_size = compressor.memory_size() + +Streaming Output API +^^^^^^^^^^^^^^^^^^^^ + +``read_from(reader)`` provides a mechanism to stream data out of a compressor +as an iterator of data chunks.:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh): + # Do something with emitted data. + +``read_from()`` accepts an object that has a ``read(size)`` method or conforms +to the buffer protocol. (``bytes`` and ``memoryview`` are 2 common types that +provide the buffer protocol.) + +Uncompressed data is fetched from the source either by calling ``read(size)`` +or by fetching a slice of data from the object directly (in the case where +the buffer protocol is being used). The returned iterator consists of chunks +of compressed data. + +Like ``write_to()``, ``read_from()`` also accepts a ``size`` argument +declaring the size of the input stream:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh, size=some_int): + pass + +You can also control the size that data is ``read()`` from the source and +the ideal size of output chunks:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh, read_size=16384, write_size=8192): + pass + +Stream Copying API +^^^^^^^^^^^^^^^^^^ + +``copy_stream(ifh, ofh)`` can be used to copy data between 2 streams while +compressing it.:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh) + +For example, say you wish to compress a file:: + + cctx = zstd.ZstdCompressor() + with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh: + cctx.copy_stream(ifh, ofh) + +It is also possible to declare the size of the source stream:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh, size=len_of_input) + +You can also specify how large the chunks that are ``read()`` and ``write()`` +from and to the streams:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh, read_size=32768, write_size=16384) + +The stream copier returns a 2-tuple of bytes read and written:: + + cctx = zstd.ZstdCompressor() + read_count, write_count = cctx.copy_stream(ifh, ofh) + +Compressor API +^^^^^^^^^^^^^^ + +``compressobj()`` returns an object that exposes ``compress(data)`` and +``flush()`` methods. Each returns compressed data or an empty bytes. + +The purpose of ``compressobj()`` is to provide an API-compatible interface +with ``zlib.compressobj`` and ``bz2.BZ2Compressor``. This allows callers to +swap in different compressor objects while using the same API. + +Once ``flush()`` is called, the compressor will no longer accept new data +to ``compress()``. ``flush()`` **must** be called to end the compression +context. If not called, the returned data may be incomplete. + +Here is how this API should be used:: + + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj() + data = cobj.compress(b'raw input 0') + data = cobj.compress(b'raw input 1') + data = cobj.flush() + +For best performance results, keep input chunks under 256KB. This avoids +extra allocations for a large output object. + +It is possible to declare the input size of the data that will be fed into +the compressor:: + + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj(size=6) + data = cobj.compress(b'foobar') + data = cobj.flush() + +ZstdDecompressor +---------------- + +The ``ZstdDecompressor`` class provides an interface for performing +decompression. + +Each instance is associated with parameters that control decompression. These +come from the following named arguments (all optional): + +dict_data + Compression dictionary to use. + +The interface of this class is very similar to ``ZstdCompressor`` (by design). + +Simple API +^^^^^^^^^^ + +``decompress(data)`` can be used to decompress an entire compressed zstd +frame in a single operation.:: + + dctx = zstd.ZstdDecompressor() + decompressed = dctx.decompress(data) + +By default, ``decompress(data)`` will only work on data written with the content +size encoded in its header. This can be achieved by creating a +``ZstdCompressor`` with ``write_content_size=True``. If compressed data without +an embedded content size is seen, ``zstd.ZstdError`` will be raised. + +If the compressed data doesn't have its content size embedded within it, +decompression can be attempted by specifying the ``max_output_size`` +argument.:: + + dctx = zstd.ZstdDecompressor() + uncompressed = dctx.decompress(data, max_output_size=1048576) + +Ideally, ``max_output_size`` will be identical to the decompressed output +size. + +If ``max_output_size`` is too small to hold the decompressed data, +``zstd.ZstdError`` will be raised. + +If ``max_output_size`` is larger than the decompressed data, the allocated +output buffer will be resized to only use the space required. + +Please note that an allocation of the requested ``max_output_size`` will be +performed every time the method is called. Setting to a very large value could +result in a lot of work for the memory allocator and may result in +``MemoryError`` being raised if the allocation fails. + +If the exact size of decompressed data is unknown, it is **strongly** +recommended to use a streaming API. + +Streaming Input API +^^^^^^^^^^^^^^^^^^^ + +``write_to(fh)`` can be used to incrementally send compressed data to a +decompressor.:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh) as decompressor: + decompressor.write(compressed_data) + +This behaves similarly to ``zstd.ZstdCompressor``: compressed data is written to +the decompressor by calling ``write(data)`` and decompressed output is written +to the output object by calling its ``write(data)`` method. + +The size of chunks being ``write()`` to the destination can be specified:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh, write_size=16384) as decompressor: + pass + +You can see how much memory is being used by the decompressor:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh) as decompressor: + byte_size = decompressor.memory_size() + +Streaming Output API +^^^^^^^^^^^^^^^^^^^^ + +``read_from(fh)`` provides a mechanism to stream decompressed data out of a +compressed source as an iterator of data chunks.:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh): + # Do something with original data. + +``read_from()`` accepts a) an object with a ``read(size)`` method that will +return compressed bytes b) an object conforming to the buffer protocol that +can expose its data as a contiguous range of bytes. The ``bytes`` and +``memoryview`` types expose this buffer protocol. + +``read_from()`` returns an iterator whose elements are chunks of the +decompressed data. + +The size of requested ``read()`` from the source can be specified:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh, read_size=16384): + pass + +It is also possible to skip leading bytes in the input data:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh, skip_bytes=1): + pass + +Skipping leading bytes is useful if the source data contains extra +*header* data but you want to avoid the overhead of making a buffer copy +or allocating a new ``memoryview`` object in order to decompress the data. + +Similarly to ``ZstdCompressor.read_from()``, the consumer of the iterator +controls when data is decompressed. If the iterator isn't consumed, +decompression is put on hold. + +When ``read_from()`` is passed an object conforming to the buffer protocol, +the behavior may seem similar to what occurs when the simple decompression +API is used. However, this API works when the decompressed size is unknown. +Furthermore, if feeding large inputs, the decompressor will work in chunks +instead of performing a single operation. + +Stream Copying API +^^^^^^^^^^^^^^^^^^ + +``copy_stream(ifh, ofh)`` can be used to copy data across 2 streams while +performing decompression.:: + + dctx = zstd.ZstdDecompressor() + dctx.copy_stream(ifh, ofh) + +e.g. to decompress a file to another file:: + + dctx = zstd.ZstdDecompressor() + with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh: + dctx.copy_stream(ifh, ofh) + +The size of chunks being ``read()`` and ``write()`` from and to the streams +can be specified:: + + dctx = zstd.ZstdDecompressor() + dctx.copy_stream(ifh, ofh, read_size=8192, write_size=16384) + +Decompressor API +^^^^^^^^^^^^^^^^ + +``decompressobj()`` returns an object that exposes a ``decompress(data)`` +methods. Compressed data chunks are fed into ``decompress(data)`` and +uncompressed output (or an empty bytes) is returned. Output from subsequent +calls needs to be concatenated to reassemble the full decompressed byte +sequence. + +The purpose of ``decompressobj()`` is to provide an API-compatible interface +with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor``. This allows callers +to swap in different decompressor objects while using the same API. + +Each object is single use: once an input frame is decoded, ``decompress()`` +can no longer be called. + +Here is how this API should be used:: + + dctx = zstd.ZstdDeompressor() + dobj = cctx.decompressobj() + data = dobj.decompress(compressed_chunk_0) + data = dobj.decompress(compressed_chunk_1) + +Choosing an API +--------------- + +Various forms of compression and decompression APIs are provided because each +are suitable for different use cases. + +The simple/one-shot APIs are useful for small data, when the decompressed +data size is known (either recorded in the zstd frame header via +``write_content_size`` or known via an out-of-band mechanism, such as a file +size). + +A limitation of the simple APIs is that input or output data must fit in memory. +And unless using advanced tricks with Python *buffer objects*, both input and +output must fit in memory simultaneously. + +Another limitation is that compression or decompression is performed as a single +operation. So if you feed large input, it could take a long time for the +function to return. + +The streaming APIs do not have the limitations of the simple API. The cost to +this is they are more complex to use than a single function call. + +The streaming APIs put the caller in control of compression and decompression +behavior by allowing them to directly control either the input or output side +of the operation. + +With the streaming input APIs, the caller feeds data into the compressor or +decompressor as they see fit. Output data will only be written after the caller +has explicitly written data. + +With the streaming output APIs, the caller consumes output from the compressor +or decompressor as they see fit. The compressor or decompressor will only +consume data from the source when the caller is ready to receive it. + +One end of the streaming APIs involves a file-like object that must +``write()`` output data or ``read()`` input data. Depending on what the +backing storage for these objects is, those operations may not complete quickly. +For example, when streaming compressed data to a file, the ``write()`` into +a streaming compressor could result in a ``write()`` to the filesystem, which +may take a long time to finish due to slow I/O on the filesystem. So, there +may be overhead in streaming APIs beyond the compression and decompression +operations. + +Dictionary Creation and Management +---------------------------------- + +Zstandard allows *dictionaries* to be used when compressing and +decompressing data. The idea is that if you are compressing a lot of similar +data, you can precompute common properties of that data (such as recurring +byte sequences) to achieve better compression ratios. + +In Python, compression dictionaries are represented as the +``ZstdCompressionDict`` type. + +Instances can be constructed from bytes:: + + dict_data = zstd.ZstdCompressionDict(data) + +More interestingly, instances can be created by *training* on sample data:: + + dict_data = zstd.train_dictionary(size, samples) + +This takes a list of bytes instances and creates and returns a +``ZstdCompressionDict``. + +You can see how many bytes are in the dictionary by calling ``len()``:: + + dict_data = zstd.train_dictionary(size, samples) + dict_size = len(dict_data) # will not be larger than ``size`` + +Once you have a dictionary, you can pass it to the objects performing +compression and decompression:: + + dict_data = zstd.train_dictionary(16384, samples) + + cctx = zstd.ZstdCompressor(dict_data=dict_data) + for source_data in input_data: + compressed = cctx.compress(source_data) + # Do something with compressed data. + + dctx = zstd.ZstdDecompressor(dict_data=dict_data) + for compressed_data in input_data: + buffer = io.BytesIO() + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed_data) + # Do something with raw data in ``buffer``. + +Dictionaries have unique integer IDs. You can retrieve this ID via:: + + dict_id = zstd.dictionary_id(dict_data) + +You can obtain the raw data in the dict (useful for persisting and constructing +a ``ZstdCompressionDict`` later) via ``as_bytes()``:: + + dict_data = zstd.train_dictionary(size, samples) + raw_data = dict_data.as_bytes() + +Explicit Compression Parameters +------------------------------- + +Zstandard's integer compression levels along with the input size and dictionary +size are converted into a data structure defining multiple parameters to tune +behavior of the compression algorithm. It is possible to use define this +data structure explicitly to have lower-level control over compression behavior. + +The ``zstd.CompressionParameters`` type represents this data structure. +You can see how Zstandard converts compression levels to this data structure +by calling ``zstd.get_compression_parameters()``. e.g.:: + + params = zstd.get_compression_parameters(5) + +This function also accepts the uncompressed data size and dictionary size +to adjust parameters:: + + params = zstd.get_compression_parameters(3, source_size=len(data), dict_size=len(dict_data)) + +You can also construct compression parameters from their low-level components:: + + params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST) + +You can then configure a compressor to use the custom parameters:: + + cctx = zstd.ZstdCompressor(compression_params=params) + +The members of the ``CompressionParameters`` tuple are as follows:: + +* 0 - Window log +* 1 - Chain log +* 2 - Hash log +* 3 - Search log +* 4 - Search length +* 5 - Target length +* 6 - Strategy (one of the ``zstd.STRATEGY_`` constants) + +You'll need to read the Zstandard documentation for what these parameters +do. + +Misc Functionality +------------------ + +estimate_compression_context_size(CompressionParameters) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given a ``CompressionParameters`` struct, estimate the memory size required +to perform compression. + +estimate_decompression_context_size() +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Estimate the memory size requirements for a decompressor instance. + +Constants +--------- + +The following module constants/attributes are exposed: + +ZSTD_VERSION + This module attribute exposes a 3-tuple of the Zstandard version. e.g. + ``(1, 0, 0)`` +MAX_COMPRESSION_LEVEL + Integer max compression level accepted by compression functions +COMPRESSION_RECOMMENDED_INPUT_SIZE + Recommended chunk size to feed to compressor functions +COMPRESSION_RECOMMENDED_OUTPUT_SIZE + Recommended chunk size for compression output +DECOMPRESSION_RECOMMENDED_INPUT_SIZE + Recommended chunk size to feed into decompresor functions +DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE + Recommended chunk size for decompression output + +FRAME_HEADER + bytes containing header of the Zstandard frame +MAGIC_NUMBER + Frame header as an integer + +WINDOWLOG_MIN + Minimum value for compression parameter +WINDOWLOG_MAX + Maximum value for compression parameter +CHAINLOG_MIN + Minimum value for compression parameter +CHAINLOG_MAX + Maximum value for compression parameter +HASHLOG_MIN + Minimum value for compression parameter +HASHLOG_MAX + Maximum value for compression parameter +SEARCHLOG_MIN + Minimum value for compression parameter +SEARCHLOG_MAX + Maximum value for compression parameter +SEARCHLENGTH_MIN + Minimum value for compression parameter +SEARCHLENGTH_MAX + Maximum value for compression parameter +TARGETLENGTH_MIN + Minimum value for compression parameter +TARGETLENGTH_MAX + Maximum value for compression parameter +STRATEGY_FAST + Compression strategory +STRATEGY_DFAST + Compression strategory +STRATEGY_GREEDY + Compression strategory +STRATEGY_LAZY + Compression strategory +STRATEGY_LAZY2 + Compression strategory +STRATEGY_BTLAZY2 + Compression strategory +STRATEGY_BTOPT + Compression strategory + +Note on Zstandard's *Experimental* API +====================================== + +Many of the Zstandard APIs used by this module are marked as *experimental* +within the Zstandard project. This includes a large number of useful +features, such as compression and frame parameters and parts of dictionary +compression. + +It is unclear how Zstandard's C API will evolve over time, especially with +regards to this *experimental* functionality. We will try to maintain +backwards compatibility at the Python API level. However, we cannot +guarantee this for things not under our control. + +Since a copy of the Zstandard source code is distributed with this +module and since we compile against it, the behavior of a specific +version of this module should be constant for all of time. So if you +pin the version of this module used in your projects (which is a Python +best practice), you should be buffered from unwanted future changes. + +Donate +====== + +A lot of time has been invested into this project by the author. + +If you find this project useful and would like to thank the author for +their work, consider donating some money. Any amount is appreciated. + +.. image:: https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif + :target: https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=gregory%2eszorc%40gmail%2ecom&lc=US&item_name=python%2dzstandard¤cy_code=USD&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted + :alt: Donate via PayPal + +.. |ci-status| image:: https://travis-ci.org/indygreg/python-zstandard.svg?branch=master + :target: https://travis-ci.org/indygreg/python-zstandard + +.. |win-ci-status| image:: https://ci.appveyor.com/api/projects/status/github/indygreg/python-zstandard?svg=true + :target: https://ci.appveyor.com/project/indygreg/python-zstandard + :alt: Windows build status
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressiondict.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,247 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = { "dict_size", "samples", "parameters", NULL }; + size_t capacity; + PyObject* samples; + Py_ssize_t samplesLen; + PyObject* parameters = NULL; + ZDICT_params_t zparams; + Py_ssize_t sampleIndex; + Py_ssize_t sampleSize; + PyObject* sampleItem; + size_t zresult; + void* sampleBuffer; + void* sampleOffset; + size_t samplesSize = 0; + size_t* sampleSizes; + void* dict; + ZstdCompressionDict* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|O!", kwlist, + &capacity, + &PyList_Type, &samples, + (PyObject*)&DictParametersType, ¶meters)) { + return NULL; + } + + /* Validate parameters first since it is easiest. */ + zparams.selectivityLevel = 0; + zparams.compressionLevel = 0; + zparams.notificationLevel = 0; + zparams.dictID = 0; + zparams.reserved[0] = 0; + zparams.reserved[1] = 0; + + if (parameters) { + /* TODO validate data ranges */ + zparams.selectivityLevel = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 0)); + zparams.compressionLevel = PyLong_AsLong(PyTuple_GetItem(parameters, 1)); + zparams.notificationLevel = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 2)); + zparams.dictID = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 3)); + } + + /* Figure out the size of the raw samples */ + samplesLen = PyList_Size(samples); + for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) { + sampleItem = PyList_GetItem(samples, sampleIndex); + if (!PyBytes_Check(sampleItem)) { + PyErr_SetString(PyExc_ValueError, "samples must be bytes"); + /* TODO probably need to perform DECREF here */ + return NULL; + } + samplesSize += PyBytes_GET_SIZE(sampleItem); + } + + /* Now that we know the total size of the raw simples, we can allocate + a buffer for the raw data */ + sampleBuffer = malloc(samplesSize); + if (!sampleBuffer) { + PyErr_NoMemory(); + return NULL; + } + sampleSizes = malloc(samplesLen * sizeof(size_t)); + if (!sampleSizes) { + free(sampleBuffer); + PyErr_NoMemory(); + return NULL; + } + + sampleOffset = sampleBuffer; + /* Now iterate again and assemble the samples in the buffer */ + for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) { + sampleItem = PyList_GetItem(samples, sampleIndex); + sampleSize = PyBytes_GET_SIZE(sampleItem); + sampleSizes[sampleIndex] = sampleSize; + memcpy(sampleOffset, PyBytes_AS_STRING(sampleItem), sampleSize); + sampleOffset = (char*)sampleOffset + sampleSize; + } + + dict = malloc(capacity); + if (!dict) { + free(sampleSizes); + free(sampleBuffer); + PyErr_NoMemory(); + return NULL; + } + + zresult = ZDICT_trainFromBuffer_advanced(dict, capacity, + sampleBuffer, sampleSizes, (unsigned int)samplesLen, + zparams); + if (ZDICT_isError(zresult)) { + PyErr_Format(ZstdError, "Cannot train dict: %s", ZDICT_getErrorName(zresult)); + free(dict); + free(sampleSizes); + free(sampleBuffer); + return NULL; + } + + result = PyObject_New(ZstdCompressionDict, &ZstdCompressionDictType); + if (!result) { + return NULL; + } + + result->dictData = dict; + result->dictSize = zresult; + return result; +} + + +PyDoc_STRVAR(ZstdCompressionDict__doc__, +"ZstdCompressionDict(data) - Represents a computed compression dictionary\n" +"\n" +"This type holds the results of a computed Zstandard compression dictionary.\n" +"Instances are obtained by calling ``train_dictionary()`` or by passing bytes\n" +"obtained from another source into the constructor.\n" +); + +static int ZstdCompressionDict_init(ZstdCompressionDict* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + + self->dictData = NULL; + self->dictSize = 0; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return -1; + } + + self->dictData = malloc(sourceSize); + if (!self->dictData) { + PyErr_NoMemory(); + return -1; + } + + memcpy(self->dictData, source, sourceSize); + self->dictSize = sourceSize; + + return 0; + } + +static void ZstdCompressionDict_dealloc(ZstdCompressionDict* self) { + if (self->dictData) { + free(self->dictData); + self->dictData = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionDict_dict_id(ZstdCompressionDict* self) { + unsigned dictID = ZDICT_getDictID(self->dictData, self->dictSize); + + return PyLong_FromLong(dictID); +} + +static PyObject* ZstdCompressionDict_as_bytes(ZstdCompressionDict* self) { + return PyBytes_FromStringAndSize(self->dictData, self->dictSize); +} + +static PyMethodDef ZstdCompressionDict_methods[] = { + { "dict_id", (PyCFunction)ZstdCompressionDict_dict_id, METH_NOARGS, + PyDoc_STR("dict_id() -- obtain the numeric dictionary ID") }, + { "as_bytes", (PyCFunction)ZstdCompressionDict_as_bytes, METH_NOARGS, + PyDoc_STR("as_bytes() -- obtain the raw bytes constituting the dictionary data") }, + { NULL, NULL } +}; + +static Py_ssize_t ZstdCompressionDict_length(ZstdCompressionDict* self) { + return self->dictSize; +} + +static PySequenceMethods ZstdCompressionDict_sq = { + (lenfunc)ZstdCompressionDict_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject ZstdCompressionDictType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionDict", /* tp_name */ + sizeof(ZstdCompressionDict), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionDict_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &ZstdCompressionDict_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressionDict__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionDict_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ZstdCompressionDict_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressiondict_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressionDictType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionDictType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdCompressionDictType); + PyModule_AddObject(mod, "ZstdCompressionDict", + (PyObject*)&ZstdCompressionDictType); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressionparams.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,226 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams) { + zparams->windowLog = params->windowLog; + zparams->chainLog = params->chainLog; + zparams->hashLog = params->hashLog; + zparams->searchLog = params->searchLog; + zparams->searchLength = params->searchLength; + zparams->targetLength = params->targetLength; + zparams->strategy = params->strategy; +} + +CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args) { + int compressionLevel; + unsigned PY_LONG_LONG sourceSize = 0; + Py_ssize_t dictSize = 0; + ZSTD_compressionParameters params; + CompressionParametersObject* result; + + if (!PyArg_ParseTuple(args, "i|Kn", &compressionLevel, &sourceSize, &dictSize)) { + return NULL; + } + + params = ZSTD_getCParams(compressionLevel, sourceSize, dictSize); + + result = PyObject_New(CompressionParametersObject, &CompressionParametersType); + if (!result) { + return NULL; + } + + result->windowLog = params.windowLog; + result->chainLog = params.chainLog; + result->hashLog = params.hashLog; + result->searchLog = params.searchLog; + result->searchLength = params.searchLength; + result->targetLength = params.targetLength; + result->strategy = params.strategy; + + return result; +} + +PyObject* estimate_compression_context_size(PyObject* self, PyObject* args) { + CompressionParametersObject* params; + ZSTD_compressionParameters zparams; + PyObject* result; + + if (!PyArg_ParseTuple(args, "O!", &CompressionParametersType, ¶ms)) { + return NULL; + } + + ztopy_compression_parameters(params, &zparams); + result = PyLong_FromSize_t(ZSTD_estimateCCtxSize(zparams)); + return result; +} + +PyDoc_STRVAR(CompressionParameters__doc__, +"CompressionParameters: low-level control over zstd compression"); + +static PyObject* CompressionParameters_new(PyTypeObject* subtype, PyObject* args, PyObject* kwargs) { + CompressionParametersObject* self; + unsigned windowLog; + unsigned chainLog; + unsigned hashLog; + unsigned searchLog; + unsigned searchLength; + unsigned targetLength; + unsigned strategy; + + if (!PyArg_ParseTuple(args, "IIIIIII", &windowLog, &chainLog, &hashLog, &searchLog, + &searchLength, &targetLength, &strategy)) { + return NULL; + } + + if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid window log value"); + return NULL; + } + + if (chainLog < ZSTD_CHAINLOG_MIN || chainLog > ZSTD_CHAINLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid chain log value"); + return NULL; + } + + if (hashLog < ZSTD_HASHLOG_MIN || hashLog > ZSTD_HASHLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid hash log value"); + return NULL; + } + + if (searchLog < ZSTD_SEARCHLOG_MIN || searchLog > ZSTD_SEARCHLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid search log value"); + return NULL; + } + + if (searchLength < ZSTD_SEARCHLENGTH_MIN || searchLength > ZSTD_SEARCHLENGTH_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid search length value"); + return NULL; + } + + if (targetLength < ZSTD_TARGETLENGTH_MIN || targetLength > ZSTD_TARGETLENGTH_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid target length value"); + return NULL; + } + + if (strategy < ZSTD_fast || strategy > ZSTD_btopt) { + PyErr_SetString(PyExc_ValueError, "invalid strategy value"); + return NULL; + } + + self = (CompressionParametersObject*)subtype->tp_alloc(subtype, 1); + if (!self) { + return NULL; + } + + self->windowLog = windowLog; + self->chainLog = chainLog; + self->hashLog = hashLog; + self->searchLog = searchLog; + self->searchLength = searchLength; + self->targetLength = targetLength; + self->strategy = strategy; + + return (PyObject*)self; +} + +static void CompressionParameters_dealloc(PyObject* self) { + PyObject_Del(self); +} + +static Py_ssize_t CompressionParameters_length(PyObject* self) { + return 7; +} + +static PyObject* CompressionParameters_item(PyObject* o, Py_ssize_t i) { + CompressionParametersObject* self = (CompressionParametersObject*)o; + + switch (i) { + case 0: + return PyLong_FromLong(self->windowLog); + case 1: + return PyLong_FromLong(self->chainLog); + case 2: + return PyLong_FromLong(self->hashLog); + case 3: + return PyLong_FromLong(self->searchLog); + case 4: + return PyLong_FromLong(self->searchLength); + case 5: + return PyLong_FromLong(self->targetLength); + case 6: + return PyLong_FromLong(self->strategy); + default: + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +static PySequenceMethods CompressionParameters_sq = { + CompressionParameters_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + CompressionParameters_item, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject CompressionParametersType = { + PyVarObject_HEAD_INIT(NULL, 0) + "CompressionParameters", /* tp_name */ + sizeof(CompressionParametersObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)CompressionParameters_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &CompressionParameters_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + CompressionParameters__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + CompressionParameters_new, /* tp_new */ +}; + +void compressionparams_module_init(PyObject* mod) { + Py_TYPE(&CompressionParametersType) = &PyType_Type; + if (PyType_Ready(&CompressionParametersType) < 0) { + return; + } + + Py_IncRef((PyObject*)&CompressionParametersType); + PyModule_AddObject(mod, "CompressionParameters", + (PyObject*)&CompressionParametersType); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressionwriter.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,235 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompresssionWriter__doc__, +"""A context manager used for writing compressed output to a writer.\n" +); + +static void ZstdCompressionWriter_dealloc(ZstdCompressionWriter* self) { + Py_XDECREF(self->compressor); + Py_XDECREF(self->writer); + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) { + if (self->entered) { + PyErr_SetString(ZstdError, "cannot __enter__ multiple times"); + return NULL; + } + + self->cstream = CStream_from_ZstdCompressor(self->compressor, self->sourceSize); + if (!self->cstream) { + return NULL; + } + + self->entered = 1; + + Py_INCREF(self); + return (PyObject*)self; +} + +static PyObject* ZstdCompressionWriter_exit(ZstdCompressionWriter* self, PyObject* args) { + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_tb; + size_t zresult; + + ZSTD_outBuffer output; + PyObject* res; + + if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) { + return NULL; + } + + self->entered = 0; + + if (self->cstream && exc_type == Py_None && exc_value == Py_None && + exc_tb == Py_None) { + + output.dst = malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + while (1) { + zresult = ZSTD_endStream(self->cstream, &output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + free(output.dst); + return NULL; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + } + + if (!zresult) { + break; + } + + output.pos = 0; + } + + free(output.dst); + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + Py_RETURN_FALSE; +} + +static PyObject* ZstdCompressionWriter_memory_size(ZstdCompressionWriter* self) { + if (!self->cstream) { + PyErr_SetString(ZstdError, "cannot determine size of an inactive compressor; " + "call when a context manager is active"); + return NULL; + } + + return PyLong_FromSize_t(ZSTD_sizeof_CStream(self->cstream)); +} + +static PyObject* ZstdCompressionWriter_write(ZstdCompressionWriter* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + PyObject* res; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + if (!self->entered) { + PyErr_SetString(ZstdError, "compress must be called from an active context manager"); + return NULL; + } + + output.dst = malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + free(output.dst); + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + /* Copy data from output buffer to writer. */ + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + } + output.pos = 0; + } + + free(output.dst); + + /* TODO return bytes written */ + Py_RETURN_NONE; + } + +static PyMethodDef ZstdCompressionWriter_methods[] = { + { "__enter__", (PyCFunction)ZstdCompressionWriter_enter, METH_NOARGS, + PyDoc_STR("Enter a compression context.") }, + { "__exit__", (PyCFunction)ZstdCompressionWriter_exit, METH_VARARGS, + PyDoc_STR("Exit a compression context.") }, + { "memory_size", (PyCFunction)ZstdCompressionWriter_memory_size, METH_NOARGS, + PyDoc_STR("Obtain the memory size of the underlying compressor") }, + { "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS, + PyDoc_STR("Compress data") }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressionWriterType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionWriter", /* tp_name */ + sizeof(ZstdCompressionWriter), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionWriter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompresssionWriter__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionWriter_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressionwriter_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressionWriterType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionWriterType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressobj.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,205 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompressionObj__doc__, +"Perform compression using a standard library compatible API.\n" +); + +static void ZstdCompressionObj_dealloc(ZstdCompressionObj* self) { + PyMem_Free(self->output.dst); + self->output.dst = NULL; + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + Py_XDECREF(self->compressor); + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionObj_compress(ZstdCompressionObj* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + ZSTD_inBuffer input; + size_t zresult; + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (self->flushed) { + PyErr_SetString(ZstdError, "cannot call compress() after flush() has been called"); + return NULL; + } + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + if (self->output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) { + return NULL; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + self->output.dst, self->output.pos); + } + else { + result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + if (!result) { + return NULL; + } + } + + self->output.pos = 0; + } + } + + if (result) { + return result; + } + else { + return PyBytes_FromString(""); + } +} + +static PyObject* ZstdCompressionObj_flush(ZstdCompressionObj* self) { + size_t zresult; + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (self->flushed) { + PyErr_SetString(ZstdError, "flush() already called"); + return NULL; + } + + self->flushed = 1; + + while (1) { + zresult = ZSTD_endStream(self->cstream, &self->output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + if (self->output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) { + return NULL; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + self->output.dst, self->output.pos); + } + else { + result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + if (!result) { + return NULL; + } + } + + self->output.pos = 0; + } + + if (!zresult) { + break; + } + } + + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + + if (result) { + return result; + } + else { + return PyBytes_FromString(""); + } +} + +static PyMethodDef ZstdCompressionObj_methods[] = { + { "compress", (PyCFunction)ZstdCompressionObj_compress, METH_VARARGS, + PyDoc_STR("compress data") }, + { "flush", (PyCFunction)ZstdCompressionObj_flush, METH_NOARGS, + PyDoc_STR("finish compression operation") }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressionObjType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionObj", /* tp_name */ + sizeof(ZstdCompressionObj), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionObj_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressionObj__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionObj_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressobj_module_init(PyObject* module) { + Py_TYPE(&ZstdCompressionObjType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionObjType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressor.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,757 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +/** +* Initialize a zstd CStream from a ZstdCompressor instance. +* +* Returns a ZSTD_CStream on success or NULL on failure. If NULL, a Python +* exception will be set. +*/ +ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize) { + ZSTD_CStream* cstream; + ZSTD_parameters zparams; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + + cstream = ZSTD_createCStream(); + if (!cstream) { + PyErr_SetString(ZstdError, "cannot create CStream"); + return NULL; + } + + if (compressor->dict) { + dictData = compressor->dict->dictData; + dictSize = compressor->dict->dictSize; + } + + memset(&zparams, 0, sizeof(zparams)); + if (compressor->cparams) { + ztopy_compression_parameters(compressor->cparams, &zparams.cParams); + /* Do NOT call ZSTD_adjustCParams() here because the compression params + come from the user. */ + } + else { + zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize); + } + + zparams.fParams = compressor->fparams; + + zresult = ZSTD_initCStream_advanced(cstream, dictData, dictSize, zparams, sourceSize); + + if (ZSTD_isError(zresult)) { + ZSTD_freeCStream(cstream); + PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + return cstream; +} + + +PyDoc_STRVAR(ZstdCompressor__doc__, +"ZstdCompressor(level=None, dict_data=None, compression_params=None)\n" +"\n" +"Create an object used to perform Zstandard compression.\n" +"\n" +"An instance can compress data various ways. Instances can be used multiple\n" +"times. Each compression operation will use the compression parameters\n" +"defined at construction time.\n" +"\n" +"Compression can be configured via the following names arguments:\n" +"\n" +"level\n" +" Integer compression level.\n" +"dict_data\n" +" A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n" +"compression_params\n" +" A ``CompressionParameters`` instance defining low-level compression" +" parameters. If defined, this will overwrite the ``level`` argument.\n" +"write_checksum\n" +" If True, a 4 byte content checksum will be written with the compressed\n" +" data, allowing the decompressor to perform content verification.\n" +"write_content_size\n" +" If True, the decompressed content size will be included in the header of\n" +" the compressed data. This data will only be written if the compressor\n" +" knows the size of the input data.\n" +"write_dict_id\n" +" Determines whether the dictionary ID will be written into the compressed\n" +" data. Defaults to True. Only adds content to the compressed data if\n" +" a dictionary is being used.\n" +); + +static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "level", + "dict_data", + "compression_params", + "write_checksum", + "write_content_size", + "write_dict_id", + NULL + }; + + int level = 3; + ZstdCompressionDict* dict = NULL; + CompressionParametersObject* params = NULL; + PyObject* writeChecksum = NULL; + PyObject* writeContentSize = NULL; + PyObject* writeDictID = NULL; + + self->dict = NULL; + self->cparams = NULL; + self->cdict = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO", kwlist, + &level, &ZstdCompressionDictType, &dict, + &CompressionParametersType, ¶ms, + &writeChecksum, &writeContentSize, &writeDictID)) { + return -1; + } + + if (level < 1) { + PyErr_SetString(PyExc_ValueError, "level must be greater than 0"); + return -1; + } + + if (level > ZSTD_maxCLevel()) { + PyErr_Format(PyExc_ValueError, "level must be less than %d", + ZSTD_maxCLevel() + 1); + return -1; + } + + self->compressionLevel = level; + + if (dict) { + self->dict = dict; + Py_INCREF(dict); + } + + if (params) { + self->cparams = params; + Py_INCREF(params); + } + + memset(&self->fparams, 0, sizeof(self->fparams)); + + if (writeChecksum && PyObject_IsTrue(writeChecksum)) { + self->fparams.checksumFlag = 1; + } + if (writeContentSize && PyObject_IsTrue(writeContentSize)) { + self->fparams.contentSizeFlag = 1; + } + if (writeDictID && PyObject_Not(writeDictID)) { + self->fparams.noDictIDFlag = 1; + } + + return 0; +} + +static void ZstdCompressor_dealloc(ZstdCompressor* self) { + Py_XDECREF(self->cparams); + Py_XDECREF(self->dict); + + if (self->cdict) { + ZSTD_freeCDict(self->cdict); + self->cdict = NULL; + } + + PyObject_Del(self); +} + +PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__, +"copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n" +"compress data between streams\n" +"\n" +"Data will be read from ``ifh``, compressed, and written to ``ofh``.\n" +"``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n" +"method.\n" +"\n" +"An optional ``size`` argument specifies the size of the source stream.\n" +"If defined, compression parameters will be tuned based on the size.\n" +"\n" +"Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n" +"of ``read()`` and ``write()`` operations, respectively. By default, they use\n" +"the default compression stream input and output sizes, respectively.\n" +); + +static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "ifh", + "ofh", + "size", + "read_size", + "write_size", + NULL + }; + + PyObject* source; + PyObject* dest; + Py_ssize_t sourceSize = 0; + size_t inSize = ZSTD_CStreamInSize(); + size_t outSize = ZSTD_CStreamOutSize(); + ZSTD_CStream* cstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t totalRead = 0; + Py_ssize_t totalWrite = 0; + char* readBuffer; + Py_ssize_t readSize; + PyObject* readResult; + PyObject* res = NULL; + size_t zresult; + PyObject* writeResult; + PyObject* totalReadPy; + PyObject* totalWritePy; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk", kwlist, &source, &dest, &sourceSize, + &inSize, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(source, "read")) { + PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); + return NULL; + } + + if (!PyObject_HasAttrString(dest, "write")) { + PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); + return NULL; + } + + cstream = CStream_from_ZstdCompressor(self, sourceSize); + if (!cstream) { + res = NULL; + goto finally; + } + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + res = NULL; + goto finally; + } + output.size = outSize; + output.pos = 0; + + while (1) { + /* Try to read from source stream. */ + readResult = PyObject_CallMethod(source, "read", "n", inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + goto finally; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + + /* If no data was read, we're at EOF. */ + if (0 == readSize) { + break; + } + + totalRead += readSize; + + /* Send data to compressor */ + input.src = readBuffer; + input.size = readSize; + input.pos = 0; + + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(cstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + res = NULL; + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(writeResult); + totalWrite += output.pos; + output.pos = 0; + } + } + } + + /* We've finished reading. Now flush the compressor stream. */ + while (1) { + zresult = ZSTD_endStream(cstream, &output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + res = NULL; + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + totalWrite += output.pos; + Py_XDECREF(writeResult); + output.pos = 0; + } + + if (!zresult) { + break; + } + } + + ZSTD_freeCStream(cstream); + cstream = NULL; + + totalReadPy = PyLong_FromSsize_t(totalRead); + totalWritePy = PyLong_FromSsize_t(totalWrite); + res = PyTuple_Pack(2, totalReadPy, totalWritePy); + Py_DecRef(totalReadPy); + Py_DecRef(totalWritePy); + +finally: + if (output.dst) { + PyMem_Free(output.dst); + } + + if (cstream) { + ZSTD_freeCStream(cstream); + } + + return res; +} + +PyDoc_STRVAR(ZstdCompressor_compress__doc__, +"compress(data)\n" +"\n" +"Compress data in a single operation.\n" +"\n" +"This is the simplest mechanism to perform compression: simply pass in a\n" +"value and get a compressed value back. It is almost the most prone to abuse.\n" +"The input and output values must fit in memory, so passing in very large\n" +"values can result in excessive memory usage. For this reason, one of the\n" +"streaming based APIs is preferred for larger values.\n" +); + +static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t destSize; + ZSTD_CCtx* cctx; + PyObject* output; + char* dest; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + ZSTD_parameters zparams; + ZSTD_customMem zmem; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + destSize = ZSTD_compressBound(sourceSize); + output = PyBytes_FromStringAndSize(NULL, destSize); + if (!output) { + return NULL; + } + + dest = PyBytes_AsString(output); + + cctx = ZSTD_createCCtx(); + if (!cctx) { + Py_DECREF(output); + PyErr_SetString(ZstdError, "could not create CCtx"); + return NULL; + } + + if (self->dict) { + dictData = self->dict->dictData; + dictSize = self->dict->dictSize; + } + + memset(&zparams, 0, sizeof(zparams)); + if (!self->cparams) { + zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize); + } + else { + ztopy_compression_parameters(self->cparams, &zparams.cParams); + /* Do NOT call ZSTD_adjustCParams() here because the compression params + come from the user. */ + } + + zparams.fParams = self->fparams; + + /* The raw dict data has to be processed before it can be used. Since this + adds overhead - especially if multiple dictionary compression operations + are performed on the same ZstdCompressor instance - we create a + ZSTD_CDict once and reuse it for all operations. */ + + /* TODO the zparams (which can be derived from the source data size) used + on first invocation are effectively reused for subsequent operations. This + may not be appropriate if input sizes vary significantly and could affect + chosen compression parameters. + https://github.com/facebook/zstd/issues/358 tracks this issue. */ + if (dictData && !self->cdict) { + Py_BEGIN_ALLOW_THREADS + memset(&zmem, 0, sizeof(zmem)); + self->cdict = ZSTD_createCDict_advanced(dictData, dictSize, zparams, zmem); + Py_END_ALLOW_THREADS + + if (!self->cdict) { + Py_DECREF(output); + ZSTD_freeCCtx(cctx); + PyErr_SetString(ZstdError, "could not create compression dictionary"); + return NULL; + } + } + + Py_BEGIN_ALLOW_THREADS + /* By avoiding ZSTD_compress(), we don't necessarily write out content + size. This means the argument to ZstdCompressor to control frame + parameters is honored. */ + if (self->cdict) { + zresult = ZSTD_compress_usingCDict(cctx, dest, destSize, + source, sourceSize, self->cdict); + } + else { + zresult = ZSTD_compress_advanced(cctx, dest, destSize, + source, sourceSize, dictData, dictSize, zparams); + } + Py_END_ALLOW_THREADS + + ZSTD_freeCCtx(cctx); + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult)); + Py_CLEAR(output); + return NULL; + } + else { + Py_SIZE(output) = zresult; + } + + return output; +} + +PyDoc_STRVAR(ZstdCompressionObj__doc__, +"compressobj()\n" +"\n" +"Return an object exposing ``compress(data)`` and ``flush()`` methods.\n" +"\n" +"The returned object exposes an API similar to ``zlib.compressobj`` and\n" +"``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n" +"without changing how compression is performed.\n" +); + +static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "size", + NULL + }; + + Py_ssize_t inSize = 0; + size_t outSize = ZSTD_CStreamOutSize(); + ZstdCompressionObj* result = PyObject_New(ZstdCompressionObj, &ZstdCompressionObjType); + if (!result) { + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &inSize)) { + return NULL; + } + + result->cstream = CStream_from_ZstdCompressor(self, inSize); + if (!result->cstream) { + Py_DECREF(result); + return NULL; + } + + result->output.dst = PyMem_Malloc(outSize); + if (!result->output.dst) { + PyErr_NoMemory(); + Py_DECREF(result); + return NULL; + } + result->output.size = outSize; + result->output.pos = 0; + + result->compressor = self; + Py_INCREF(result->compressor); + + result->flushed = 0; + + return result; +} + +PyDoc_STRVAR(ZstdCompressor_read_from__doc__, +"read_from(reader, [size=0, read_size=default, write_size=default])\n" +"Read uncompress data from a reader and return an iterator\n" +"\n" +"Returns an iterator of compressed data produced from reading from ``reader``.\n" +"\n" +"Uncompressed data will be obtained from ``reader`` by calling the\n" +"``read(size)`` method of it. The source data will be streamed into a\n" +"compressor. As compressed data is available, it will be exposed to the\n" +"iterator.\n" +"\n" +"Data is read from the source in chunks of ``read_size``. Compressed chunks\n" +"are at most ``write_size`` bytes. Both values default to the zstd input and\n" +"and output defaults, respectively.\n" +"\n" +"The caller is partially in control of how fast data is fed into the\n" +"compressor by how it consumes the returned iterator. The compressor will\n" +"not consume from the reader unless the caller consumes from the iterator.\n" +); + +static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "reader", + "size", + "read_size", + "write_size", + NULL + }; + + PyObject* reader; + Py_ssize_t sourceSize = 0; + size_t inSize = ZSTD_CStreamInSize(); + size_t outSize = ZSTD_CStreamOutSize(); + ZstdCompressorIterator* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk", kwlist, &reader, &sourceSize, + &inSize, &outSize)) { + return NULL; + } + + result = PyObject_New(ZstdCompressorIterator, &ZstdCompressorIteratorType); + if (!result) { + return NULL; + } + + result->compressor = NULL; + result->reader = NULL; + result->buffer = NULL; + result->cstream = NULL; + result->input.src = NULL; + result->output.dst = NULL; + result->readResult = NULL; + + if (PyObject_HasAttrString(reader, "read")) { + result->reader = reader; + Py_INCREF(result->reader); + } + else if (1 == PyObject_CheckBuffer(reader)) { + result->buffer = PyMem_Malloc(sizeof(Py_buffer)); + if (!result->buffer) { + goto except; + } + + memset(result->buffer, 0, sizeof(Py_buffer)); + + if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { + goto except; + } + + result->bufferOffset = 0; + sourceSize = result->buffer->len; + } + else { + PyErr_SetString(PyExc_ValueError, + "must pass an object with a read() method or conforms to buffer protocol"); + goto except; + } + + result->compressor = self; + Py_INCREF(result->compressor); + + result->sourceSize = sourceSize; + result->cstream = CStream_from_ZstdCompressor(self, sourceSize); + if (!result->cstream) { + goto except; + } + + result->inSize = inSize; + result->outSize = outSize; + + result->output.dst = PyMem_Malloc(outSize); + if (!result->output.dst) { + PyErr_NoMemory(); + goto except; + } + result->output.size = outSize; + result->output.pos = 0; + + result->input.src = NULL; + result->input.size = 0; + result->input.pos = 0; + + result->finishedInput = 0; + result->finishedOutput = 0; + + goto finally; + +except: + if (result->cstream) { + ZSTD_freeCStream(result->cstream); + result->cstream = NULL; + } + + Py_DecRef((PyObject*)result->compressor); + Py_DecRef(result->reader); + + Py_DECREF(result); + result = NULL; + +finally: + return result; +} + +PyDoc_STRVAR(ZstdCompressor_write_to___doc__, +"Create a context manager to write compressed data to an object.\n" +"\n" +"The passed object must have a ``write()`` method.\n" +"\n" +"The caller feeds input data to the object by calling ``compress(data)``.\n" +"Compressed data is written to the argument given to this function.\n" +"\n" +"The function takes an optional ``size`` argument indicating the total size\n" +"of the eventual input. If specified, the size will influence compression\n" +"parameter tuning and could result in the size being written into the\n" +"header of the compressed data.\n" +"\n" +"An optional ``write_size`` argument is also accepted. It defines the maximum\n" +"byte size of chunks fed to ``write()``. By default, it uses the zstd default\n" +"for a compressor output stream.\n" +); + +static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "writer", + "size", + "write_size", + NULL + }; + + PyObject* writer; + ZstdCompressionWriter* result; + Py_ssize_t sourceSize = 0; + size_t outSize = ZSTD_CStreamOutSize(); + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk", kwlist, &writer, &sourceSize, + &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(writer, "write")) { + PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); + return NULL; + } + + result = PyObject_New(ZstdCompressionWriter, &ZstdCompressionWriterType); + if (!result) { + return NULL; + } + + result->compressor = self; + Py_INCREF(result->compressor); + + result->writer = writer; + Py_INCREF(result->writer); + + result->sourceSize = sourceSize; + + result->outSize = outSize; + + result->entered = 0; + result->cstream = NULL; + + return result; +} + +static PyMethodDef ZstdCompressor_methods[] = { + { "compress", (PyCFunction)ZstdCompressor_compress, METH_VARARGS, + ZstdCompressor_compress__doc__ }, + { "compressobj", (PyCFunction)ZstdCompressor_compressobj, + METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ }, + { "copy_stream", (PyCFunction)ZstdCompressor_copy_stream, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ }, + { "read_from", (PyCFunction)ZstdCompressor_read_from, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_from__doc__ }, + { "write_to", (PyCFunction)ZstdCompressor_write_to, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_write_to___doc__ }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressor", /* tp_name */ + sizeof(ZstdCompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ZstdCompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressor_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressorType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressorType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdCompressorType); + PyModule_AddObject(mod, "ZstdCompressor", + (PyObject*)&ZstdCompressorType); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/compressoriterator.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,234 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompressorIterator__doc__, +"Represents an iterator of compressed data.\n" +); + +static void ZstdCompressorIterator_dealloc(ZstdCompressorIterator* self) { + Py_XDECREF(self->readResult); + Py_XDECREF(self->compressor); + Py_XDECREF(self->reader); + + if (self->buffer) { + PyBuffer_Release(self->buffer); + PyMem_FREE(self->buffer); + self->buffer = NULL; + } + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + if (self->output.dst) { + PyMem_Free(self->output.dst); + self->output.dst = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressorIterator_iter(PyObject* self) { + Py_INCREF(self); + return self; +} + +static PyObject* ZstdCompressorIterator_iternext(ZstdCompressorIterator* self) { + size_t zresult; + PyObject* readResult = NULL; + PyObject* chunk; + char* readBuffer; + Py_ssize_t readSize = 0; + Py_ssize_t bufferRemaining; + + if (self->finishedOutput) { + PyErr_SetString(PyExc_StopIteration, "output flushed"); + return NULL; + } + +feedcompressor: + + /* If we have data left in the input, consume it. */ + if (self->input.pos < self->input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* Release the Python object holding the input buffer. */ + if (self->input.pos == self->input.size) { + self->input.src = NULL; + self->input.pos = 0; + self->input.size = 0; + Py_DECREF(self->readResult); + self->readResult = NULL; + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + /* If it produced output data, emit it. */ + if (self->output.pos) { + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; + } + } + + /* We should never have output data sitting around after a previous call. */ + assert(self->output.pos == 0); + + /* The code above should have either emitted a chunk and returned or consumed + the entire input buffer. So the state of the input buffer is not + relevant. */ + if (!self->finishedInput) { + if (self->reader) { + readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + return NULL; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + } + else { + assert(self->buffer && self->buffer->buf); + + /* Only support contiguous C arrays. */ + assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL); + assert(self->buffer->itemsize == 1); + + readBuffer = (char*)self->buffer->buf + self->bufferOffset; + bufferRemaining = self->buffer->len - self->bufferOffset; + readSize = min(bufferRemaining, (Py_ssize_t)self->inSize); + self->bufferOffset += readSize; + } + + if (0 == readSize) { + Py_XDECREF(readResult); + self->finishedInput = 1; + } + else { + self->readResult = readResult; + } + } + + /* EOF */ + if (0 == readSize) { + zresult = ZSTD_endStream(self->cstream, &self->output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + assert(self->output.pos); + + if (0 == zresult) { + self->finishedOutput = 1; + } + + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; + } + + /* New data from reader. Feed into compressor. */ + self->input.src = readBuffer; + self->input.size = readSize; + self->input.pos = 0; + + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* The input buffer currently points to memory managed by Python + (readBuffer). This object was allocated by this function. If it wasn't + fully consumed, we need to release it in a subsequent function call. + If it is fully consumed, do that now. + */ + if (self->input.pos == self->input.size) { + self->input.src = NULL; + self->input.pos = 0; + self->input.size = 0; + Py_XDECREF(self->readResult); + self->readResult = NULL; + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + assert(self->input.pos <= self->input.size); + + /* If we didn't write anything, start the process over. */ + if (0 == self->output.pos) { + goto feedcompressor; + } + + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; +} + +PyTypeObject ZstdCompressorIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressorIterator", /* tp_name */ + sizeof(ZstdCompressorIterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressorIterator_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressorIterator__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + ZstdCompressorIterator_iter, /* tp_iter */ + (iternextfunc)ZstdCompressorIterator_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressoriterator_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressorIteratorType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressorIteratorType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/constants.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,84 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +static char frame_header[] = { + '\x28', + '\xb5', + '\x2f', + '\xfd', +}; + +void constants_module_init(PyObject* mod) { + PyObject* version; + PyObject* zstdVersion; + PyObject* frameHeader; + +#if PY_MAJOR_VERSION >= 3 + version = PyUnicode_FromString(PYTHON_ZSTANDARD_VERSION); +#else + version = PyString_FromString(PYTHON_ZSTANDARD_VERSION); +#endif + Py_INCREF(version); + PyModule_AddObject(mod, "__version__", version); + + ZstdError = PyErr_NewException("zstd.ZstdError", NULL, NULL); + PyModule_AddObject(mod, "ZstdError", ZstdError); + + /* For now, the version is a simple tuple instead of a dedicated type. */ + zstdVersion = PyTuple_New(3); + PyTuple_SetItem(zstdVersion, 0, PyLong_FromLong(ZSTD_VERSION_MAJOR)); + PyTuple_SetItem(zstdVersion, 1, PyLong_FromLong(ZSTD_VERSION_MINOR)); + PyTuple_SetItem(zstdVersion, 2, PyLong_FromLong(ZSTD_VERSION_RELEASE)); + Py_IncRef(zstdVersion); + PyModule_AddObject(mod, "ZSTD_VERSION", zstdVersion); + + frameHeader = PyBytes_FromStringAndSize(frame_header, sizeof(frame_header)); + if (frameHeader) { + PyModule_AddObject(mod, "FRAME_HEADER", frameHeader); + } + else { + PyErr_Format(PyExc_ValueError, "could not create frame header object"); + } + + PyModule_AddIntConstant(mod, "MAX_COMPRESSION_LEVEL", ZSTD_maxCLevel()); + PyModule_AddIntConstant(mod, "COMPRESSION_RECOMMENDED_INPUT_SIZE", + (long)ZSTD_CStreamInSize()); + PyModule_AddIntConstant(mod, "COMPRESSION_RECOMMENDED_OUTPUT_SIZE", + (long)ZSTD_CStreamOutSize()); + PyModule_AddIntConstant(mod, "DECOMPRESSION_RECOMMENDED_INPUT_SIZE", + (long)ZSTD_DStreamInSize()); + PyModule_AddIntConstant(mod, "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE", + (long)ZSTD_DStreamOutSize()); + + PyModule_AddIntConstant(mod, "MAGIC_NUMBER", ZSTD_MAGICNUMBER); + PyModule_AddIntConstant(mod, "WINDOWLOG_MIN", ZSTD_WINDOWLOG_MIN); + PyModule_AddIntConstant(mod, "WINDOWLOG_MAX", ZSTD_WINDOWLOG_MAX); + PyModule_AddIntConstant(mod, "CHAINLOG_MIN", ZSTD_CHAINLOG_MIN); + PyModule_AddIntConstant(mod, "CHAINLOG_MAX", ZSTD_CHAINLOG_MAX); + PyModule_AddIntConstant(mod, "HASHLOG_MIN", ZSTD_HASHLOG_MIN); + PyModule_AddIntConstant(mod, "HASHLOG_MAX", ZSTD_HASHLOG_MAX); + PyModule_AddIntConstant(mod, "HASHLOG3_MAX", ZSTD_HASHLOG3_MAX); + PyModule_AddIntConstant(mod, "SEARCHLOG_MIN", ZSTD_SEARCHLOG_MIN); + PyModule_AddIntConstant(mod, "SEARCHLOG_MAX", ZSTD_SEARCHLOG_MAX); + PyModule_AddIntConstant(mod, "SEARCHLENGTH_MIN", ZSTD_SEARCHLENGTH_MIN); + PyModule_AddIntConstant(mod, "SEARCHLENGTH_MAX", ZSTD_SEARCHLENGTH_MAX); + PyModule_AddIntConstant(mod, "TARGETLENGTH_MIN", ZSTD_TARGETLENGTH_MIN); + PyModule_AddIntConstant(mod, "TARGETLENGTH_MAX", ZSTD_TARGETLENGTH_MAX); + + PyModule_AddIntConstant(mod, "STRATEGY_FAST", ZSTD_fast); + PyModule_AddIntConstant(mod, "STRATEGY_DFAST", ZSTD_dfast); + PyModule_AddIntConstant(mod, "STRATEGY_GREEDY", ZSTD_greedy); + PyModule_AddIntConstant(mod, "STRATEGY_LAZY", ZSTD_lazy); + PyModule_AddIntConstant(mod, "STRATEGY_LAZY2", ZSTD_lazy2); + PyModule_AddIntConstant(mod, "STRATEGY_BTLAZY2", ZSTD_btlazy2); + PyModule_AddIntConstant(mod, "STRATEGY_BTOPT", ZSTD_btopt); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/decompressionwriter.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,187 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdDecompressionWriter__doc, +"""A context manager used for writing decompressed output.\n" +); + +static void ZstdDecompressionWriter_dealloc(ZstdDecompressionWriter* self) { + Py_XDECREF(self->decompressor); + Py_XDECREF(self->writer); + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdDecompressionWriter_enter(ZstdDecompressionWriter* self) { + if (self->entered) { + PyErr_SetString(ZstdError, "cannot __enter__ multiple times"); + return NULL; + } + + self->dstream = DStream_from_ZstdDecompressor(self->decompressor); + if (!self->dstream) { + return NULL; + } + + self->entered = 1; + + Py_INCREF(self); + return (PyObject*)self; +} + +static PyObject* ZstdDecompressionWriter_exit(ZstdDecompressionWriter* self, PyObject* args) { + self->entered = 0; + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + Py_RETURN_FALSE; +} + +static PyObject* ZstdDecompressionWriter_memory_size(ZstdDecompressionWriter* self) { + if (!self->dstream) { + PyErr_SetString(ZstdError, "cannot determine size of inactive decompressor; " + "call when context manager is active"); + return NULL; + } + + return PyLong_FromSize_t(ZSTD_sizeof_DStream(self->dstream)); +} + +static PyObject* ZstdDecompressionWriter_write(ZstdDecompressionWriter* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult = 0; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + PyObject* res; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + if (!self->entered) { + PyErr_SetString(ZstdError, "write must be called from an active context manager"); + return NULL; + } + + output.dst = malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + free(output.dst); + PyErr_Format(ZstdError, "zstd decompress error: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + output.pos = 0; + } + } + + free(output.dst); + + /* TODO return bytes written */ + Py_RETURN_NONE; + } + +static PyMethodDef ZstdDecompressionWriter_methods[] = { + { "__enter__", (PyCFunction)ZstdDecompressionWriter_enter, METH_NOARGS, + PyDoc_STR("Enter a decompression context.") }, + { "__exit__", (PyCFunction)ZstdDecompressionWriter_exit, METH_VARARGS, + PyDoc_STR("Exit a decompression context.") }, + { "memory_size", (PyCFunction)ZstdDecompressionWriter_memory_size, METH_NOARGS, + PyDoc_STR("Obtain the memory size in bytes of the underlying decompressor.") }, + { "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS, + PyDoc_STR("Compress data") }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressionWriterType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressionWriter", /* tp_name */ + sizeof(ZstdDecompressionWriter),/* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdDecompressionWriter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdDecompressionWriter__doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdDecompressionWriter_methods,/* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressionwriter_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressionWriterType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressionWriterType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/decompressobj.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,170 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(DecompressionObj__doc__, +"Perform decompression using a standard library compatible API.\n" +); + +static void DecompressionObj_dealloc(ZstdDecompressionObj* self) { + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + Py_XDECREF(self->decompressor); + + PyObject_Del(self); +} + +static PyObject* DecompressionObj_decompress(ZstdDecompressionObj* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t outSize = ZSTD_DStreamOutSize(); + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (self->finished) { + PyErr_SetString(ZstdError, "cannot use a decompressobj multiple times"); + return NULL; + } + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", +#else + if (!PyArg_ParseTuple(args, "s#", +#endif + &source, &sourceSize)) { + return NULL; + } + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + return NULL; + } + output.size = outSize; + output.pos = 0; + + /* Read input until exhausted. */ + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd decompressor error: %s", + ZSTD_getErrorName(zresult)); + result = NULL; + goto finally; + } + + if (0 == zresult) { + self->finished = 1; + } + + if (output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + output.pos)) { + goto except; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + output.dst, output.pos); + } + else { + result = PyBytes_FromStringAndSize(output.dst, output.pos); + if (!result) { + goto except; + } + } + + output.pos = 0; + } + } + + if (!result) { + result = PyBytes_FromString(""); + } + + goto finally; + +except: + Py_DecRef(result); + result = NULL; + +finally: + PyMem_Free(output.dst); + + return result; +} + +static PyMethodDef DecompressionObj_methods[] = { + { "decompress", (PyCFunction)DecompressionObj_decompress, + METH_VARARGS, PyDoc_STR("decompress data") }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressionObjType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressionObj", /* tp_name */ + sizeof(ZstdDecompressionObj), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)DecompressionObj_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + DecompressionObj__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DecompressionObj_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressobj_module_init(PyObject* module) { + Py_TYPE(&ZstdDecompressionObjType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressionObjType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/decompressor.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,669 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor) { + ZSTD_DStream* dstream; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + + dstream = ZSTD_createDStream(); + if (!dstream) { + PyErr_SetString(ZstdError, "could not create DStream"); + return NULL; + } + + if (decompressor->dict) { + dictData = decompressor->dict->dictData; + dictSize = decompressor->dict->dictSize; + } + + if (dictData) { + zresult = ZSTD_initDStream_usingDict(dstream, dictData, dictSize); + } + else { + zresult = ZSTD_initDStream(dstream); + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "could not initialize DStream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + return dstream; +} + +PyDoc_STRVAR(Decompressor__doc__, +"ZstdDecompressor(dict_data=None)\n" +"\n" +"Create an object used to perform Zstandard decompression.\n" +"\n" +"An instance can perform multiple decompression operations." +); + +static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "dict_data", + NULL + }; + + ZstdCompressionDict* dict = NULL; + + self->refdctx = NULL; + self->dict = NULL; + self->ddict = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!", kwlist, + &ZstdCompressionDictType, &dict)) { + return -1; + } + + /* Instead of creating a ZSTD_DCtx for every decompression operation, + we create an instance at object creation time and recycle it via + ZSTD_copyDCTx() on each use. This means each use is a malloc+memcpy + instead of a malloc+init. */ + /* TODO lazily initialize the reference ZSTD_DCtx on first use since + not instances of ZstdDecompressor will use a ZSTD_DCtx. */ + self->refdctx = ZSTD_createDCtx(); + if (!self->refdctx) { + PyErr_NoMemory(); + goto except; + } + + if (dict) { + self->dict = dict; + Py_INCREF(dict); + } + + return 0; + +except: + if (self->refdctx) { + ZSTD_freeDCtx(self->refdctx); + self->refdctx = NULL; + } + + return -1; +} + +static void Decompressor_dealloc(ZstdDecompressor* self) { + if (self->refdctx) { + ZSTD_freeDCtx(self->refdctx); + } + + Py_XDECREF(self->dict); + + if (self->ddict) { + ZSTD_freeDDict(self->ddict); + self->ddict = NULL; + } + + PyObject_Del(self); +} + +PyDoc_STRVAR(Decompressor_copy_stream__doc__, + "copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n" + "\n" + "Compressed data will be read from ``ifh``, decompressed, and written to\n" + "``ofh``. ``ifh`` must have a ``read(size)`` method. ``ofh`` must have a\n" + "``write(data)`` method.\n" + "\n" + "The optional ``read_size`` and ``write_size`` arguments control the chunk\n" + "size of data that is ``read()`` and ``write()`` between streams. They default\n" + "to the default input and output sizes of zstd decompressor streams.\n" +); + +static PyObject* Decompressor_copy_stream(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "ifh", + "ofh", + "read_size", + "write_size", + NULL + }; + + PyObject* source; + PyObject* dest; + size_t inSize = ZSTD_DStreamInSize(); + size_t outSize = ZSTD_DStreamOutSize(); + ZSTD_DStream* dstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t totalRead = 0; + Py_ssize_t totalWrite = 0; + char* readBuffer; + Py_ssize_t readSize; + PyObject* readResult; + PyObject* res = NULL; + size_t zresult = 0; + PyObject* writeResult; + PyObject* totalReadPy; + PyObject* totalWritePy; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk", kwlist, &source, + &dest, &inSize, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(source, "read")) { + PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); + return NULL; + } + + if (!PyObject_HasAttrString(dest, "write")) { + PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); + return NULL; + } + + dstream = DStream_from_ZstdDecompressor(self); + if (!dstream) { + res = NULL; + goto finally; + } + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + res = NULL; + goto finally; + } + output.size = outSize; + output.pos = 0; + + /* Read source stream until EOF */ + while (1) { + readResult = PyObject_CallMethod(source, "read", "n", inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + goto finally; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + + /* If no data was read, we're at EOF. */ + if (0 == readSize) { + break; + } + + totalRead += readSize; + + /* Send data to decompressor */ + input.src = readBuffer; + input.size = readSize; + input.pos = 0; + + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd decompressor error: %s", + ZSTD_getErrorName(zresult)); + res = NULL; + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + + Py_XDECREF(writeResult); + totalWrite += output.pos; + output.pos = 0; + } + } + } + + /* Source stream is exhausted. Finish up. */ + + ZSTD_freeDStream(dstream); + dstream = NULL; + + totalReadPy = PyLong_FromSsize_t(totalRead); + totalWritePy = PyLong_FromSsize_t(totalWrite); + res = PyTuple_Pack(2, totalReadPy, totalWritePy); + Py_DecRef(totalReadPy); + Py_DecRef(totalWritePy); + + finally: + if (output.dst) { + PyMem_Free(output.dst); + } + + if (dstream) { + ZSTD_freeDStream(dstream); + } + + return res; +} + +PyDoc_STRVAR(Decompressor_decompress__doc__, +"decompress(data[, max_output_size=None]) -- Decompress data in its entirety\n" +"\n" +"This method will decompress the entirety of the argument and return the\n" +"result.\n" +"\n" +"The input bytes are expected to contain a full Zstandard frame (something\n" +"compressed with ``ZstdCompressor.compress()`` or similar). If the input does\n" +"not contain a full frame, an exception will be raised.\n" +"\n" +"If the frame header of the compressed data does not contain the content size\n" +"``max_output_size`` must be specified or ``ZstdError`` will be raised. An\n" +"allocation of size ``max_output_size`` will be performed and an attempt will\n" +"be made to perform decompression into that buffer. If the buffer is too\n" +"small or cannot be allocated, ``ZstdError`` will be raised. The buffer will\n" +"be resized if it is too large.\n" +"\n" +"Uncompressed data could be much larger than compressed data. As a result,\n" +"calling this function could result in a very large memory allocation being\n" +"performed to hold the uncompressed data. Therefore it is **highly**\n" +"recommended to use a streaming decompression method instead of this one.\n" +); + +PyObject* Decompressor_decompress(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "data", + "max_output_size", + NULL + }; + + const char* source; + Py_ssize_t sourceSize; + Py_ssize_t maxOutputSize = 0; + unsigned long long decompressedSize; + size_t destCapacity; + PyObject* result = NULL; + ZSTD_DCtx* dctx = NULL; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n", kwlist, +#else + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n", kwlist, +#endif + &source, &sourceSize, &maxOutputSize)) { + return NULL; + } + + dctx = PyMem_Malloc(ZSTD_sizeof_DCtx(self->refdctx)); + if (!dctx) { + PyErr_NoMemory(); + return NULL; + } + + ZSTD_copyDCtx(dctx, self->refdctx); + + if (self->dict) { + dictData = self->dict->dictData; + dictSize = self->dict->dictSize; + } + + if (dictData && !self->ddict) { + Py_BEGIN_ALLOW_THREADS + self->ddict = ZSTD_createDDict(dictData, dictSize); + Py_END_ALLOW_THREADS + + if (!self->ddict) { + PyErr_SetString(ZstdError, "could not create decompression dict"); + goto except; + } + } + + decompressedSize = ZSTD_getDecompressedSize(source, sourceSize); + /* 0 returned if content size not in the zstd frame header */ + if (0 == decompressedSize) { + if (0 == maxOutputSize) { + PyErr_SetString(ZstdError, "input data invalid or missing content size " + "in frame header"); + goto except; + } + else { + result = PyBytes_FromStringAndSize(NULL, maxOutputSize); + destCapacity = maxOutputSize; + } + } + else { + result = PyBytes_FromStringAndSize(NULL, decompressedSize); + destCapacity = decompressedSize; + } + + if (!result) { + goto except; + } + + Py_BEGIN_ALLOW_THREADS + if (self->ddict) { + zresult = ZSTD_decompress_usingDDict(dctx, PyBytes_AsString(result), destCapacity, + source, sourceSize, self->ddict); + } + else { + zresult = ZSTD_decompressDCtx(dctx, PyBytes_AsString(result), destCapacity, source, sourceSize); + } + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult)); + goto except; + } + else if (decompressedSize && zresult != decompressedSize) { + PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu", + zresult, decompressedSize); + goto except; + } + else if (zresult < destCapacity) { + if (_PyBytes_Resize(&result, zresult)) { + goto except; + } + } + + goto finally; + +except: + Py_DecRef(result); + result = NULL; + +finally: + if (dctx) { + PyMem_FREE(dctx); + } + + return result; +} + +PyDoc_STRVAR(Decompressor_decompressobj__doc__, +"decompressobj()\n" +"\n" +"Incrementally feed data into a decompressor.\n" +"\n" +"The returned object exposes a ``decompress(data)`` method. This makes it\n" +"compatible with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor`` so that\n" +"callers can swap in the zstd decompressor while using the same API.\n" +); + +static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self) { + ZstdDecompressionObj* result = PyObject_New(ZstdDecompressionObj, &ZstdDecompressionObjType); + if (!result) { + return NULL; + } + + result->dstream = DStream_from_ZstdDecompressor(self); + if (!result->dstream) { + Py_DecRef((PyObject*)result); + return NULL; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->finished = 0; + + return result; +} + +PyDoc_STRVAR(Decompressor_read_from__doc__, +"read_from(reader[, read_size=default, write_size=default, skip_bytes=0])\n" +"Read compressed data and return an iterator\n" +"\n" +"Returns an iterator of decompressed data chunks produced from reading from\n" +"the ``reader``.\n" +"\n" +"Compressed data will be obtained from ``reader`` by calling the\n" +"``read(size)`` method of it. The source data will be streamed into a\n" +"decompressor. As decompressed data is available, it will be exposed to the\n" +"returned iterator.\n" +"\n" +"Data is ``read()`` in chunks of size ``read_size`` and exposed to the\n" +"iterator in chunks of size ``write_size``. The default values are the input\n" +"and output sizes for a zstd streaming decompressor.\n" +"\n" +"There is also support for skipping the first ``skip_bytes`` of data from\n" +"the source.\n" +); + +static ZstdDecompressorIterator* Decompressor_read_from(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "reader", + "read_size", + "write_size", + "skip_bytes", + NULL + }; + + PyObject* reader; + size_t inSize = ZSTD_DStreamInSize(); + size_t outSize = ZSTD_DStreamOutSize(); + ZstdDecompressorIterator* result; + size_t skipBytes = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk", kwlist, &reader, + &inSize, &outSize, &skipBytes)) { + return NULL; + } + + if (skipBytes >= inSize) { + PyErr_SetString(PyExc_ValueError, + "skip_bytes must be smaller than read_size"); + return NULL; + } + + result = PyObject_New(ZstdDecompressorIterator, &ZstdDecompressorIteratorType); + if (!result) { + return NULL; + } + + result->decompressor = NULL; + result->reader = NULL; + result->buffer = NULL; + result->dstream = NULL; + result->input.src = NULL; + result->output.dst = NULL; + + if (PyObject_HasAttrString(reader, "read")) { + result->reader = reader; + Py_INCREF(result->reader); + } + else if (1 == PyObject_CheckBuffer(reader)) { + /* Object claims it is a buffer. Try to get a handle to it. */ + result->buffer = PyMem_Malloc(sizeof(Py_buffer)); + if (!result->buffer) { + goto except; + } + + memset(result->buffer, 0, sizeof(Py_buffer)); + + if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { + goto except; + } + + result->bufferOffset = 0; + } + else { + PyErr_SetString(PyExc_ValueError, + "must pass an object with a read() method or conforms to buffer protocol"); + goto except; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->inSize = inSize; + result->outSize = outSize; + result->skipBytes = skipBytes; + + result->dstream = DStream_from_ZstdDecompressor(self); + if (!result->dstream) { + goto except; + } + + result->input.src = PyMem_Malloc(inSize); + if (!result->input.src) { + PyErr_NoMemory(); + goto except; + } + result->input.size = 0; + result->input.pos = 0; + + result->output.dst = NULL; + result->output.size = 0; + result->output.pos = 0; + + result->readCount = 0; + result->finishedInput = 0; + result->finishedOutput = 0; + + goto finally; + +except: + if (result->reader) { + Py_DECREF(result->reader); + result->reader = NULL; + } + + if (result->buffer) { + PyBuffer_Release(result->buffer); + Py_DECREF(result->buffer); + result->buffer = NULL; + } + + Py_DECREF(result); + result = NULL; + +finally: + + return result; +} + +PyDoc_STRVAR(Decompressor_write_to__doc__, +"Create a context manager to write decompressed data to an object.\n" +"\n" +"The passed object must have a ``write()`` method.\n" +"\n" +"The caller feeds intput data to the object by calling ``write(data)``.\n" +"Decompressed data is written to the argument given as it is decompressed.\n" +"\n" +"An optional ``write_size`` argument defines the size of chunks to\n" +"``write()`` to the writer. It defaults to the default output size for a zstd\n" +"streaming decompressor.\n" +); + +static ZstdDecompressionWriter* Decompressor_write_to(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "writer", + "write_size", + NULL + }; + + PyObject* writer; + size_t outSize = ZSTD_DStreamOutSize(); + ZstdDecompressionWriter* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k", kwlist, &writer, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(writer, "write")) { + PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); + return NULL; + } + + result = PyObject_New(ZstdDecompressionWriter, &ZstdDecompressionWriterType); + if (!result) { + return NULL; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->writer = writer; + Py_INCREF(result->writer); + + result->outSize = outSize; + + result->entered = 0; + result->dstream = NULL; + + return result; +} + +static PyMethodDef Decompressor_methods[] = { + { "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS, + Decompressor_copy_stream__doc__ }, + { "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS, + Decompressor_decompress__doc__ }, + { "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_NOARGS, + Decompressor_decompressobj__doc__ }, + { "read_from", (PyCFunction)Decompressor_read_from, METH_VARARGS | METH_KEYWORDS, + Decompressor_read_from__doc__ }, + { "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS, + Decompressor_write_to__doc__ }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressor", /* tp_name */ + sizeof(ZstdDecompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)Decompressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Decompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Decompressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Decompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressor_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressorType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressorType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdDecompressorType); + PyModule_AddObject(mod, "ZstdDecompressor", + (PyObject*)&ZstdDecompressorType); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/decompressoriterator.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,254 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdDecompressorIterator__doc__, +"Represents an iterator of decompressed data.\n" +); + +static void ZstdDecompressorIterator_dealloc(ZstdDecompressorIterator* self) { + Py_XDECREF(self->decompressor); + Py_XDECREF(self->reader); + + if (self->buffer) { + PyBuffer_Release(self->buffer); + PyMem_FREE(self->buffer); + self->buffer = NULL; + } + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + if (self->input.src) { + PyMem_Free((void*)self->input.src); + self->input.src = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdDecompressorIterator_iter(PyObject* self) { + Py_INCREF(self); + return self; +} + +static DecompressorIteratorResult read_decompressor_iterator(ZstdDecompressorIterator* self) { + size_t zresult; + PyObject* chunk; + DecompressorIteratorResult result; + size_t oldInputPos = self->input.pos; + + result.chunk = NULL; + + chunk = PyBytes_FromStringAndSize(NULL, self->outSize); + if (!chunk) { + result.errored = 1; + return result; + } + + self->output.dst = PyBytes_AsString(chunk); + self->output.size = self->outSize; + self->output.pos = 0; + + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* We're done with the pointer. Nullify to prevent anyone from getting a + handle on a Python object. */ + self->output.dst = NULL; + + if (ZSTD_isError(zresult)) { + Py_DECREF(chunk); + PyErr_Format(ZstdError, "zstd decompress error: %s", + ZSTD_getErrorName(zresult)); + result.errored = 1; + return result; + } + + self->readCount += self->input.pos - oldInputPos; + + /* Frame is fully decoded. Input exhausted and output sitting in buffer. */ + if (0 == zresult) { + self->finishedInput = 1; + self->finishedOutput = 1; + } + + /* If it produced output data, return it. */ + if (self->output.pos) { + if (self->output.pos < self->outSize) { + if (_PyBytes_Resize(&chunk, self->output.pos)) { + result.errored = 1; + return result; + } + } + } + else { + Py_DECREF(chunk); + chunk = NULL; + } + + result.errored = 0; + result.chunk = chunk; + + return result; +} + +static PyObject* ZstdDecompressorIterator_iternext(ZstdDecompressorIterator* self) { + PyObject* readResult = NULL; + char* readBuffer; + Py_ssize_t readSize; + Py_ssize_t bufferRemaining; + DecompressorIteratorResult result; + + if (self->finishedOutput) { + PyErr_SetString(PyExc_StopIteration, "output flushed"); + return NULL; + } + + /* If we have data left in the input, consume it. */ + if (self->input.pos < self->input.size) { + result = read_decompressor_iterator(self); + if (result.chunk || result.errored) { + return result.chunk; + } + + /* Else fall through to get more data from input. */ + } + +read_from_source: + + if (!self->finishedInput) { + if (self->reader) { + readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize); + if (!readResult) { + return NULL; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + } + else { + assert(self->buffer && self->buffer->buf); + + /* Only support contiguous C arrays for now */ + assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL); + assert(self->buffer->itemsize == 1); + + /* TODO avoid memcpy() below */ + readBuffer = (char *)self->buffer->buf + self->bufferOffset; + bufferRemaining = self->buffer->len - self->bufferOffset; + readSize = min(bufferRemaining, (Py_ssize_t)self->inSize); + self->bufferOffset += readSize; + } + + if (readSize) { + if (!self->readCount && self->skipBytes) { + assert(self->skipBytes < self->inSize); + if ((Py_ssize_t)self->skipBytes >= readSize) { + PyErr_SetString(PyExc_ValueError, + "skip_bytes larger than first input chunk; " + "this scenario is currently unsupported"); + Py_DecRef(readResult); + return NULL; + } + + readBuffer = readBuffer + self->skipBytes; + readSize -= self->skipBytes; + } + + /* Copy input into previously allocated buffer because it can live longer + than a single function call and we don't want to keep a ref to a Python + object around. This could be changed... */ + memcpy((void*)self->input.src, readBuffer, readSize); + self->input.size = readSize; + self->input.pos = 0; + } + /* No bytes on first read must mean an empty input stream. */ + else if (!self->readCount) { + self->finishedInput = 1; + self->finishedOutput = 1; + Py_DecRef(readResult); + PyErr_SetString(PyExc_StopIteration, "empty input"); + return NULL; + } + else { + self->finishedInput = 1; + } + + /* We've copied the data managed by memory. Discard the Python object. */ + Py_DecRef(readResult); + } + + result = read_decompressor_iterator(self); + if (result.errored || result.chunk) { + return result.chunk; + } + + /* No new output data. Try again unless we know there is no more data. */ + if (!self->finishedInput) { + goto read_from_source; + } + + PyErr_SetString(PyExc_StopIteration, "input exhausted"); + return NULL; +} + +PyTypeObject ZstdDecompressorIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressorIterator", /* tp_name */ + sizeof(ZstdDecompressorIterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdDecompressorIterator_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdDecompressorIterator__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + ZstdDecompressorIterator_iter, /* tp_iter */ + (iternextfunc)ZstdDecompressorIterator_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressoriterator_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressorIteratorType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressorIteratorType) < 0) { + return; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/dictparams.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,125 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +PyDoc_STRVAR(DictParameters__doc__, +"DictParameters: low-level control over dictionary generation"); + +static PyObject* DictParameters_new(PyTypeObject* subtype, PyObject* args, PyObject* kwargs) { + DictParametersObject* self; + unsigned selectivityLevel; + int compressionLevel; + unsigned notificationLevel; + unsigned dictID; + + if (!PyArg_ParseTuple(args, "IiII", &selectivityLevel, &compressionLevel, + ¬ificationLevel, &dictID)) { + return NULL; + } + + self = (DictParametersObject*)subtype->tp_alloc(subtype, 1); + if (!self) { + return NULL; + } + + self->selectivityLevel = selectivityLevel; + self->compressionLevel = compressionLevel; + self->notificationLevel = notificationLevel; + self->dictID = dictID; + + return (PyObject*)self; +} + +static void DictParameters_dealloc(PyObject* self) { + PyObject_Del(self); +} + +static Py_ssize_t DictParameters_length(PyObject* self) { + return 4; +} + +static PyObject* DictParameters_item(PyObject* o, Py_ssize_t i) { + DictParametersObject* self = (DictParametersObject*)o; + + switch (i) { + case 0: + return PyLong_FromLong(self->selectivityLevel); + case 1: + return PyLong_FromLong(self->compressionLevel); + case 2: + return PyLong_FromLong(self->notificationLevel); + case 3: + return PyLong_FromLong(self->dictID); + default: + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +static PySequenceMethods DictParameters_sq = { + DictParameters_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + DictParameters_item, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject DictParametersType = { + PyVarObject_HEAD_INIT(NULL, 0) + "DictParameters", /* tp_name */ + sizeof(DictParametersObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)DictParameters_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &DictParameters_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + DictParameters__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + DictParameters_new, /* tp_new */ +}; + +void dictparams_module_init(PyObject* mod) { + Py_TYPE(&DictParametersType) = &PyType_Type; + if (PyType_Ready(&DictParametersType) < 0) { + return; + } + + Py_IncRef((PyObject*)&DictParametersType); + PyModule_AddObject(mod, "DictParameters", (PyObject*)&DictParametersType); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/c-ext/python-zstandard.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,172 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#define ZSTD_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#include "mem.h" +#include "zstd.h" +#include "zdict.h" + +#define PYTHON_ZSTANDARD_VERSION "0.5.0" + +typedef struct { + PyObject_HEAD + unsigned windowLog; + unsigned chainLog; + unsigned hashLog; + unsigned searchLog; + unsigned searchLength; + unsigned targetLength; + ZSTD_strategy strategy; +} CompressionParametersObject; + +extern PyTypeObject CompressionParametersType; + +typedef struct { + PyObject_HEAD + unsigned selectivityLevel; + int compressionLevel; + unsigned notificationLevel; + unsigned dictID; +} DictParametersObject; + +extern PyTypeObject DictParametersType; + +typedef struct { + PyObject_HEAD + + void* dictData; + size_t dictSize; +} ZstdCompressionDict; + +extern PyTypeObject ZstdCompressionDictType; + +typedef struct { + PyObject_HEAD + + int compressionLevel; + ZstdCompressionDict* dict; + ZSTD_CDict* cdict; + CompressionParametersObject* cparams; + ZSTD_frameParameters fparams; +} ZstdCompressor; + +extern PyTypeObject ZstdCompressorType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + ZSTD_CStream* cstream; + ZSTD_outBuffer output; + int flushed; +} ZstdCompressionObj; + +extern PyTypeObject ZstdCompressionObjType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + PyObject* writer; + Py_ssize_t sourceSize; + size_t outSize; + ZSTD_CStream* cstream; + int entered; +} ZstdCompressionWriter; + +extern PyTypeObject ZstdCompressionWriterType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + PyObject* reader; + Py_buffer* buffer; + Py_ssize_t bufferOffset; + Py_ssize_t sourceSize; + size_t inSize; + size_t outSize; + + ZSTD_CStream* cstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + int finishedOutput; + int finishedInput; + PyObject* readResult; +} ZstdCompressorIterator; + +extern PyTypeObject ZstdCompressorIteratorType; + +typedef struct { + PyObject_HEAD + + ZSTD_DCtx* refdctx; + + ZstdCompressionDict* dict; + ZSTD_DDict* ddict; +} ZstdDecompressor; + +extern PyTypeObject ZstdDecompressorType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + ZSTD_DStream* dstream; + int finished; +} ZstdDecompressionObj; + +extern PyTypeObject ZstdDecompressionObjType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + PyObject* writer; + size_t outSize; + ZSTD_DStream* dstream; + int entered; +} ZstdDecompressionWriter; + +extern PyTypeObject ZstdDecompressionWriterType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + PyObject* reader; + Py_buffer* buffer; + Py_ssize_t bufferOffset; + size_t inSize; + size_t outSize; + size_t skipBytes; + ZSTD_DStream* dstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t readCount; + int finishedInput; + int finishedOutput; +} ZstdDecompressorIterator; + +extern PyTypeObject ZstdDecompressorIteratorType; + +typedef struct { + int errored; + PyObject* chunk; +} DecompressorIteratorResult; + +void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams); +CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args); +PyObject* estimate_compression_context_size(PyObject* self, PyObject* args); +ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize); +ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor); +ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/make_cffi.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,110 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +from __future__ import absolute_import + +import cffi +import os + + +HERE = os.path.abspath(os.path.dirname(__file__)) + +SOURCES = ['zstd/%s' % p for p in ( + 'common/entropy_common.c', + 'common/error_private.c', + 'common/fse_decompress.c', + 'common/xxhash.c', + 'common/zstd_common.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'compress/zbuff_compress.c', + 'compress/zstd_compress.c', + 'decompress/huf_decompress.c', + 'decompress/zbuff_decompress.c', + 'decompress/zstd_decompress.c', + 'dictBuilder/divsufsort.c', + 'dictBuilder/zdict.c', +)] + +INCLUDE_DIRS = [os.path.join(HERE, d) for d in ( + 'zstd', + 'zstd/common', + 'zstd/compress', + 'zstd/decompress', + 'zstd/dictBuilder', +)] + +with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: + zstd_h = fh.read() + +ffi = cffi.FFI() +ffi.set_source('_zstd_cffi', ''' +/* needed for typedefs like U32 references in zstd.h */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +''', + sources=SOURCES, include_dirs=INCLUDE_DIRS) + +# Rather than define the API definitions from zstd.h inline, munge the +# source in a way that cdef() will accept. +lines = zstd_h.splitlines() +lines = [l.rstrip() for l in lines if l.strip()] + +# Strip preprocessor directives - they aren't important for our needs. +lines = [l for l in lines + if not l.startswith((b'#if', b'#else', b'#endif', b'#include'))] + +# Remove extern C block +lines = [l for l in lines if l not in (b'extern "C" {', b'}')] + +# The version #defines don't parse and aren't necessary. Strip them. +lines = [l for l in lines if not l.startswith(( + b'#define ZSTD_H_235446', + b'#define ZSTD_LIB_VERSION', + b'#define ZSTD_QUOTE', + b'#define ZSTD_EXPAND_AND_QUOTE', + b'#define ZSTD_VERSION_STRING', + b'#define ZSTD_VERSION_NUMBER'))] + +# The C parser also doesn't like some constant defines referencing +# other constants. +# TODO we pick the 64-bit constants here. We should assert somewhere +# we're compiling for 64-bit. +def fix_constants(l): + if l.startswith(b'#define ZSTD_WINDOWLOG_MAX '): + return b'#define ZSTD_WINDOWLOG_MAX 27' + elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): + return b'#define ZSTD_CHAINLOG_MAX 28' + elif l.startswith(b'#define ZSTD_HASHLOG_MAX '): + return b'#define ZSTD_HASHLOG_MAX 27' + elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): + return b'#define ZSTD_CHAINLOG_MAX 28' + elif l.startswith(b'#define ZSTD_CHAINLOG_MIN '): + return b'#define ZSTD_CHAINLOG_MIN 6' + elif l.startswith(b'#define ZSTD_SEARCHLOG_MAX '): + return b'#define ZSTD_SEARCHLOG_MAX 26' + elif l.startswith(b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX '): + return b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX 131072' + else: + return l +lines = map(fix_constants, lines) + +# ZSTDLIB_API isn't handled correctly. Strip it. +lines = [l for l in lines if not l.startswith(b'# define ZSTDLIB_API')] +def strip_api(l): + if l.startswith(b'ZSTDLIB_API '): + return l[len(b'ZSTDLIB_API '):] + else: + return l +lines = map(strip_api, lines) + +source = b'\n'.join(lines) +ffi.cdef(source.decode('latin1')) + + +if __name__ == '__main__': + ffi.compile()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/setup.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +from setuptools import setup + +try: + import cffi +except ImportError: + cffi = None + +import setup_zstd + +# Code for obtaining the Extension instance is in its own module to +# facilitate reuse in other projects. +extensions = [setup_zstd.get_c_extension()] + +if cffi: + import make_cffi + extensions.append(make_cffi.ffi.distutils_extension()) + +version = None + +with open('c-ext/python-zstandard.h', 'r') as fh: + for line in fh: + if not line.startswith('#define PYTHON_ZSTANDARD_VERSION'): + continue + + version = line.split()[2][1:-1] + break + +if not version: + raise Exception('could not resolve package version; ' + 'this should never happen') + +setup( + name='zstandard', + version=version, + description='Zstandard bindings for Python', + long_description=open('README.rst', 'r').read(), + url='https://github.com/indygreg/python-zstandard', + author='Gregory Szorc', + author_email='gregory.szorc@gmail.com', + license='BSD', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: C', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + keywords='zstandard zstd compression', + ext_modules=extensions, + test_suite='tests', +)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/setup_zstd.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,64 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os +from distutils.extension import Extension + + +zstd_sources = ['zstd/%s' % p for p in ( + 'common/entropy_common.c', + 'common/error_private.c', + 'common/fse_decompress.c', + 'common/xxhash.c', + 'common/zstd_common.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'compress/zbuff_compress.c', + 'compress/zstd_compress.c', + 'decompress/huf_decompress.c', + 'decompress/zbuff_decompress.c', + 'decompress/zstd_decompress.c', + 'dictBuilder/divsufsort.c', + 'dictBuilder/zdict.c', +)] + + +zstd_includes = [ + 'c-ext', + 'zstd', + 'zstd/common', + 'zstd/compress', + 'zstd/decompress', + 'zstd/dictBuilder', +] + +ext_sources = [ + 'zstd.c', + 'c-ext/compressiondict.c', + 'c-ext/compressobj.c', + 'c-ext/compressor.c', + 'c-ext/compressoriterator.c', + 'c-ext/compressionparams.c', + 'c-ext/compressionwriter.c', + 'c-ext/constants.c', + 'c-ext/decompressobj.c', + 'c-ext/decompressor.c', + 'c-ext/decompressoriterator.c', + 'c-ext/decompressionwriter.c', + 'c-ext/dictparams.c', +] + + +def get_c_extension(name='zstd'): + """Obtain a distutils.extension.Extension for the C extension.""" + root = os.path.abspath(os.path.dirname(__file__)) + + sources = [os.path.join(root, p) for p in zstd_sources + ext_sources] + include_dirs = [os.path.join(root, d) for d in zstd_includes] + + # TODO compile with optimizations. + return Extension(name, sources, + include_dirs=include_dirs)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/common.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,15 @@ +import io + +class OpCountingBytesIO(io.BytesIO): + def __init__(self, *args, **kwargs): + self._read_count = 0 + self._write_count = 0 + return super(OpCountingBytesIO, self).__init__(*args, **kwargs) + + def read(self, *args): + self._read_count += 1 + return super(OpCountingBytesIO, self).read(*args) + + def write(self, data): + self._write_count += 1 + return super(OpCountingBytesIO, self).write(data)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_cffi.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,35 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +try: + import zstd_cffi +except ImportError: + raise unittest.SkipTest('cffi version of zstd not available') + + +class TestCFFIWriteToToCDecompressor(unittest.TestCase): + def test_simple(self): + orig = io.BytesIO() + orig.write(b'foo') + orig.write(b'bar') + orig.write(b'foobar' * 16384) + + dest = io.BytesIO() + cctx = zstd_cffi.ZstdCompressor() + with cctx.write_to(dest) as compressor: + compressor.write(orig.getvalue()) + + uncompressed = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(uncompressed) as decompressor: + decompressor.write(dest.getvalue()) + + self.assertEqual(uncompressed.getvalue(), orig.getvalue()) + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_compressor.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,465 @@ +import hashlib +import io +import struct +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +from .common import OpCountingBytesIO + + +if sys.version_info[0] >= 3: + next = lambda it: it.__next__() +else: + next = lambda it: it.next() + + +class TestCompressor(unittest.TestCase): + def test_level_bounds(self): + with self.assertRaises(ValueError): + zstd.ZstdCompressor(level=0) + + with self.assertRaises(ValueError): + zstd.ZstdCompressor(level=23) + + +class TestCompressor_compress(unittest.TestCase): + def test_compress_empty(self): + cctx = zstd.ZstdCompressor(level=1) + cctx.compress(b'') + + cctx = zstd.ZstdCompressor(level=22) + cctx.compress(b'') + + def test_compress_empty(self): + cctx = zstd.ZstdCompressor(level=1) + self.assertEqual(cctx.compress(b''), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_compress_large(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + + cctx = zstd.ZstdCompressor(level=3) + result = cctx.compress(b''.join(chunks)) + self.assertEqual(len(result), 999) + self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd') + + def test_write_checksum(self): + cctx = zstd.ZstdCompressor(level=1) + no_checksum = cctx.compress(b'foobar') + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + with_checksum = cctx.compress(b'foobar') + + self.assertEqual(len(with_checksum), len(no_checksum) + 4) + + def test_write_content_size(self): + cctx = zstd.ZstdCompressor(level=1) + no_size = cctx.compress(b'foobar' * 256) + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + with_size = cctx.compress(b'foobar' * 256) + + self.assertEqual(len(with_size), len(no_size) + 1) + + def test_no_dict_id(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(1024, samples) + + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + with_dict_id = cctx.compress(b'foobarfoobar') + + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False) + no_dict_id = cctx.compress(b'foobarfoobar') + + self.assertEqual(len(with_dict_id), len(no_dict_id) + 4) + + def test_compress_dict_multiple(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + + for i in range(32): + cctx.compress(b'foo bar foobar foo bar foobar') + + +class TestCompressor_compressobj(unittest.TestCase): + def test_compressobj_empty(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj() + self.assertEqual(cobj.compress(b''), b'') + self.assertEqual(cobj.flush(), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_compressobj_large(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + + cctx = zstd.ZstdCompressor(level=3) + cobj = cctx.compressobj() + + result = cobj.compress(b''.join(chunks)) + cobj.flush() + self.assertEqual(len(result), 999) + self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd') + + def test_write_checksum(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj() + no_checksum = cobj.compress(b'foobar') + cobj.flush() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + cobj = cctx.compressobj() + with_checksum = cobj.compress(b'foobar') + cobj.flush() + + self.assertEqual(len(with_checksum), len(no_checksum) + 4) + + def test_write_content_size(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj(size=len(b'foobar' * 256)) + no_size = cobj.compress(b'foobar' * 256) + cobj.flush() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + cobj = cctx.compressobj(size=len(b'foobar' * 256)) + with_size = cobj.compress(b'foobar' * 256) + cobj.flush() + + self.assertEqual(len(with_size), len(no_size) + 1) + + def test_compress_after_flush(self): + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj() + + cobj.compress(b'foo') + cobj.flush() + + with self.assertRaisesRegexp(zstd.ZstdError, 'cannot call compress\(\) after flush'): + cobj.compress(b'foo') + + with self.assertRaisesRegexp(zstd.ZstdError, 'flush\(\) already called'): + cobj.flush() + + +class TestCompressor_copy_stream(unittest.TestCase): + def test_no_read(self): + source = object() + dest = io.BytesIO() + + cctx = zstd.ZstdCompressor() + with self.assertRaises(ValueError): + cctx.copy_stream(source, dest) + + def test_no_write(self): + source = io.BytesIO() + dest = object() + + cctx = zstd.ZstdCompressor() + with self.assertRaises(ValueError): + cctx.copy_stream(source, dest) + + def test_empty(self): + source = io.BytesIO() + dest = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + r, w = cctx.copy_stream(source, dest) + self.assertEqual(int(r), 0) + self.assertEqual(w, 9) + + self.assertEqual(dest.getvalue(), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_large_data(self): + source = io.BytesIO() + for i in range(255): + source.write(struct.Struct('>B').pack(i) * 16384) + source.seek(0) + + dest = io.BytesIO() + cctx = zstd.ZstdCompressor() + r, w = cctx.copy_stream(source, dest) + + self.assertEqual(r, 255 * 16384) + self.assertEqual(w, 999) + + def test_write_checksum(self): + source = io.BytesIO(b'foobar') + no_checksum = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + cctx.copy_stream(source, no_checksum) + + source.seek(0) + with_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + cctx.copy_stream(source, with_checksum) + + self.assertEqual(len(with_checksum.getvalue()), + len(no_checksum.getvalue()) + 4) + + def test_write_content_size(self): + source = io.BytesIO(b'foobar' * 256) + no_size = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + cctx.copy_stream(source, no_size) + + source.seek(0) + with_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + cctx.copy_stream(source, with_size) + + # Source content size is unknown, so no content size written. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue())) + + source.seek(0) + with_size = io.BytesIO() + cctx.copy_stream(source, with_size, size=len(source.getvalue())) + + # We specified source size, so content size header is present. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue()) + 1) + + def test_read_write_size(self): + source = OpCountingBytesIO(b'foobarfoobar') + dest = OpCountingBytesIO() + cctx = zstd.ZstdCompressor() + r, w = cctx.copy_stream(source, dest, read_size=1, write_size=1) + + self.assertEqual(r, len(source.getvalue())) + self.assertEqual(w, 21) + self.assertEqual(source._read_count, len(source.getvalue()) + 1) + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +def compress(data, level): + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=level) + with cctx.write_to(buffer) as compressor: + compressor.write(data) + return buffer.getvalue() + + +class TestCompressor_write_to(unittest.TestCase): + def test_empty(self): + self.assertEqual(compress(b'', 1), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_multiple_compress(self): + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=5) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'x' * 8192) + + result = buffer.getvalue() + self.assertEqual(result, + b'\x28\xb5\x2f\xfd\x00\x50\x75\x00\x00\x38\x66\x6f' + b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23') + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=9, dict_data=d) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foo' * 16384) + + compressed = buffer.getvalue() + h = hashlib.sha1(compressed).hexdigest() + self.assertEqual(h, '1c5bcd25181bcd8c1a73ea8773323e0056129f92') + + def test_compression_params(self): + params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST) + + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(compression_params=params) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foobar' * 16384) + + compressed = buffer.getvalue() + h = hashlib.sha1(compressed).hexdigest() + self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99') + + def test_write_checksum(self): + no_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(no_checksum) as compressor: + compressor.write(b'foobar') + + with_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + with cctx.write_to(with_checksum) as compressor: + compressor.write(b'foobar') + + self.assertEqual(len(with_checksum.getvalue()), + len(no_checksum.getvalue()) + 4) + + def test_write_content_size(self): + no_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(no_size) as compressor: + compressor.write(b'foobar' * 256) + + with_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + with cctx.write_to(with_size) as compressor: + compressor.write(b'foobar' * 256) + + # Source size is not known in streaming mode, so header not + # written. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue())) + + # Declaring size will write the header. + with_size = io.BytesIO() + with cctx.write_to(with_size, size=len(b'foobar' * 256)) as compressor: + compressor.write(b'foobar' * 256) + + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue()) + 1) + + def test_no_dict_id(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(1024, samples) + + with_dict_id = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + with cctx.write_to(with_dict_id) as compressor: + compressor.write(b'foobarfoobar') + + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False) + no_dict_id = io.BytesIO() + with cctx.write_to(no_dict_id) as compressor: + compressor.write(b'foobarfoobar') + + self.assertEqual(len(with_dict_id.getvalue()), + len(no_dict_id.getvalue()) + 4) + + def test_memory_size(self): + cctx = zstd.ZstdCompressor(level=3) + buffer = io.BytesIO() + with cctx.write_to(buffer) as compressor: + size = compressor.memory_size() + + self.assertGreater(size, 100000) + + def test_write_size(self): + cctx = zstd.ZstdCompressor(level=3) + dest = OpCountingBytesIO() + with cctx.write_to(dest, write_size=1) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foobar') + + self.assertEqual(len(dest.getvalue()), dest._write_count) + + +class TestCompressor_read_from(unittest.TestCase): + def test_type_validation(self): + cctx = zstd.ZstdCompressor() + + # Object with read() works. + cctx.read_from(io.BytesIO()) + + # Buffer protocol works. + cctx.read_from(b'foobar') + + with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'): + cctx.read_from(True) + + def test_read_empty(self): + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + it = cctx.read_from(source) + chunks = list(it) + self.assertEqual(len(chunks), 1) + compressed = b''.join(chunks) + self.assertEqual(compressed, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + # And again with the buffer protocol. + it = cctx.read_from(b'') + chunks = list(it) + self.assertEqual(len(chunks), 1) + compressed2 = b''.join(chunks) + self.assertEqual(compressed2, compressed) + + def test_read_large(self): + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + source.write(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE) + source.write(b'o') + source.seek(0) + + # Creating an iterator should not perform any compression until + # first read. + it = cctx.read_from(source, size=len(source.getvalue())) + self.assertEqual(source.tell(), 0) + + # We should have exactly 2 output chunks. + chunks = [] + chunk = next(it) + self.assertIsNotNone(chunk) + self.assertEqual(source.tell(), zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE) + chunks.append(chunk) + chunk = next(it) + self.assertIsNotNone(chunk) + chunks.append(chunk) + + self.assertEqual(source.tell(), len(source.getvalue())) + + with self.assertRaises(StopIteration): + next(it) + + # And again for good measure. + with self.assertRaises(StopIteration): + next(it) + + # We should get the same output as the one-shot compression mechanism. + self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue())) + + # Now check the buffer protocol. + it = cctx.read_from(source.getvalue()) + chunks = list(it) + self.assertEqual(len(chunks), 2) + self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue())) + + def test_read_write_size(self): + source = OpCountingBytesIO(b'foobarfoobar') + cctx = zstd.ZstdCompressor(level=3) + for chunk in cctx.read_from(source, read_size=1, write_size=1): + self.assertEqual(len(chunk), 1) + + self.assertEqual(source._read_count, len(source.getvalue()) + 1)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_data_structures.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,107 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +try: + import hypothesis + import hypothesis.strategies as strategies +except ImportError: + hypothesis = None + +import zstd + +class TestCompressionParameters(unittest.TestCase): + def test_init_bad_arg_type(self): + with self.assertRaises(TypeError): + zstd.CompressionParameters() + + with self.assertRaises(TypeError): + zstd.CompressionParameters(0, 1) + + def test_bounds(self): + zstd.CompressionParameters(zstd.WINDOWLOG_MIN, + zstd.CHAINLOG_MIN, + zstd.HASHLOG_MIN, + zstd.SEARCHLOG_MIN, + zstd.SEARCHLENGTH_MIN, + zstd.TARGETLENGTH_MIN, + zstd.STRATEGY_FAST) + + zstd.CompressionParameters(zstd.WINDOWLOG_MAX, + zstd.CHAINLOG_MAX, + zstd.HASHLOG_MAX, + zstd.SEARCHLOG_MAX, + zstd.SEARCHLENGTH_MAX, + zstd.TARGETLENGTH_MAX, + zstd.STRATEGY_BTOPT) + + def test_get_compression_parameters(self): + p = zstd.get_compression_parameters(1) + self.assertIsInstance(p, zstd.CompressionParameters) + + self.assertEqual(p[0], 19) + +if hypothesis: + s_windowlog = strategies.integers(min_value=zstd.WINDOWLOG_MIN, + max_value=zstd.WINDOWLOG_MAX) + s_chainlog = strategies.integers(min_value=zstd.CHAINLOG_MIN, + max_value=zstd.CHAINLOG_MAX) + s_hashlog = strategies.integers(min_value=zstd.HASHLOG_MIN, + max_value=zstd.HASHLOG_MAX) + s_searchlog = strategies.integers(min_value=zstd.SEARCHLOG_MIN, + max_value=zstd.SEARCHLOG_MAX) + s_searchlength = strategies.integers(min_value=zstd.SEARCHLENGTH_MIN, + max_value=zstd.SEARCHLENGTH_MAX) + s_targetlength = strategies.integers(min_value=zstd.TARGETLENGTH_MIN, + max_value=zstd.TARGETLENGTH_MAX) + s_strategy = strategies.sampled_from((zstd.STRATEGY_FAST, + zstd.STRATEGY_DFAST, + zstd.STRATEGY_GREEDY, + zstd.STRATEGY_LAZY, + zstd.STRATEGY_LAZY2, + zstd.STRATEGY_BTLAZY2, + zstd.STRATEGY_BTOPT)) + + class TestCompressionParametersHypothesis(unittest.TestCase): + @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog, + s_searchlength, s_targetlength, s_strategy) + def test_valid_init(self, windowlog, chainlog, hashlog, searchlog, + searchlength, targetlength, strategy): + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + self.assertEqual(tuple(p), + (windowlog, chainlog, hashlog, searchlog, + searchlength, targetlength, strategy)) + + # Verify we can instantiate a compressor with the supplied values. + # ZSTD_checkCParams moves the goal posts on us from what's advertised + # in the constants. So move along with them. + if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): + searchlength += 1 + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: + searchlength -= 1 + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + + cctx = zstd.ZstdCompressor(compression_params=p) + with cctx.write_to(io.BytesIO()): + pass + + @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog, + s_searchlength, s_targetlength, s_strategy) + def test_estimate_compression_context_size(self, windowlog, chainlog, + hashlog, searchlog, + searchlength, targetlength, + strategy): + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + size = zstd.estimate_compression_context_size(p)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_decompressor.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,478 @@ +import io +import random +import struct +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +from .common import OpCountingBytesIO + + +if sys.version_info[0] >= 3: + next = lambda it: it.__next__() +else: + next = lambda it: it.next() + + +class TestDecompressor_decompress(unittest.TestCase): + def test_empty_input(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(b'') + + def test_invalid_input(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(b'foobar') + + def test_no_content_size_in_frame(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(compressed) + + def test_content_size_present(self): + cctx = zstd.ZstdCompressor(write_content_size=True) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + decompressed = dctx.decompress(compressed) + self.assertEqual(decompressed, b'foobar') + + def test_max_output_size(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + source = b'foobar' * 256 + compressed = cctx.compress(source) + + dctx = zstd.ZstdDecompressor() + # Will fit into buffer exactly the size of input. + decompressed = dctx.decompress(compressed, max_output_size=len(source)) + self.assertEqual(decompressed, source) + + # Input size - 1 fails + with self.assertRaisesRegexp(zstd.ZstdError, 'Destination buffer is too small'): + dctx.decompress(compressed, max_output_size=len(source) - 1) + + # Input size + 1 works + decompressed = dctx.decompress(compressed, max_output_size=len(source) + 1) + self.assertEqual(decompressed, source) + + # A much larger buffer works. + decompressed = dctx.decompress(compressed, max_output_size=len(source) * 64) + self.assertEqual(decompressed, source) + + def test_stupidly_large_output_buffer(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar' * 256) + dctx = zstd.ZstdDecompressor() + + # Will get OverflowError on some Python distributions that can't + # handle really large integers. + with self.assertRaises((MemoryError, OverflowError)): + dctx.decompress(compressed, max_output_size=2**62) + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + orig = b'foobar' * 16384 + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True) + compressed = cctx.compress(orig) + + dctx = zstd.ZstdDecompressor(dict_data=d) + decompressed = dctx.decompress(compressed) + + self.assertEqual(decompressed, orig) + + def test_dictionary_multiple(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + sources = (b'foobar' * 8192, b'foo' * 8192, b'bar' * 8192) + compressed = [] + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True) + for source in sources: + compressed.append(cctx.compress(source)) + + dctx = zstd.ZstdDecompressor(dict_data=d) + for i in range(len(sources)): + decompressed = dctx.decompress(compressed[i]) + self.assertEqual(decompressed, sources[i]) + + +class TestDecompressor_copy_stream(unittest.TestCase): + def test_no_read(self): + source = object() + dest = io.BytesIO() + + dctx = zstd.ZstdDecompressor() + with self.assertRaises(ValueError): + dctx.copy_stream(source, dest) + + def test_no_write(self): + source = io.BytesIO() + dest = object() + + dctx = zstd.ZstdDecompressor() + with self.assertRaises(ValueError): + dctx.copy_stream(source, dest) + + def test_empty(self): + source = io.BytesIO() + dest = io.BytesIO() + + dctx = zstd.ZstdDecompressor() + # TODO should this raise an error? + r, w = dctx.copy_stream(source, dest) + + self.assertEqual(r, 0) + self.assertEqual(w, 0) + self.assertEqual(dest.getvalue(), b'') + + def test_large_data(self): + source = io.BytesIO() + for i in range(255): + source.write(struct.Struct('>B').pack(i) * 16384) + source.seek(0) + + compressed = io.BytesIO() + cctx = zstd.ZstdCompressor() + cctx.copy_stream(source, compressed) + + compressed.seek(0) + dest = io.BytesIO() + dctx = zstd.ZstdDecompressor() + r, w = dctx.copy_stream(compressed, dest) + + self.assertEqual(r, len(compressed.getvalue())) + self.assertEqual(w, len(source.getvalue())) + + def test_read_write_size(self): + source = OpCountingBytesIO(zstd.ZstdCompressor().compress( + b'foobarfoobar')) + + dest = OpCountingBytesIO() + dctx = zstd.ZstdDecompressor() + r, w = dctx.copy_stream(source, dest, read_size=1, write_size=1) + + self.assertEqual(r, len(source.getvalue())) + self.assertEqual(w, len(b'foobarfoobar')) + self.assertEqual(source._read_count, len(source.getvalue()) + 1) + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +class TestDecompressor_decompressobj(unittest.TestCase): + def test_simple(self): + data = zstd.ZstdCompressor(level=1).compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + dobj = dctx.decompressobj() + self.assertEqual(dobj.decompress(data), b'foobar') + + def test_reuse(self): + data = zstd.ZstdCompressor(level=1).compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + dobj = dctx.decompressobj() + dobj.decompress(data) + + with self.assertRaisesRegexp(zstd.ZstdError, 'cannot use a decompressobj'): + dobj.decompress(data) + + +def decompress_via_writer(data): + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + decompressor.write(data) + return buffer.getvalue() + + +class TestDecompressor_write_to(unittest.TestCase): + def test_empty_roundtrip(self): + cctx = zstd.ZstdCompressor() + empty = cctx.compress(b'') + self.assertEqual(decompress_via_writer(empty), b'') + + def test_large_roundtrip(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + orig = b''.join(chunks) + cctx = zstd.ZstdCompressor() + compressed = cctx.compress(orig) + + self.assertEqual(decompress_via_writer(compressed), orig) + + def test_multiple_calls(self): + chunks = [] + for i in range(255): + for j in range(255): + chunks.append(struct.Struct('>B').pack(j) * i) + + orig = b''.join(chunks) + cctx = zstd.ZstdCompressor() + compressed = cctx.compress(orig) + + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + pos = 0 + while pos < len(compressed): + pos2 = pos + 8192 + decompressor.write(compressed[pos:pos2]) + pos += 8192 + self.assertEqual(buffer.getvalue(), orig) + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + orig = b'foobar' * 16384 + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(dict_data=d) + with cctx.write_to(buffer) as compressor: + compressor.write(orig) + + compressed = buffer.getvalue() + buffer = io.BytesIO() + + dctx = zstd.ZstdDecompressor(dict_data=d) + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed) + + self.assertEqual(buffer.getvalue(), orig) + + def test_memory_size(self): + dctx = zstd.ZstdDecompressor() + buffer = io.BytesIO() + with dctx.write_to(buffer) as decompressor: + size = decompressor.memory_size() + + self.assertGreater(size, 100000) + + def test_write_size(self): + source = zstd.ZstdCompressor().compress(b'foobarfoobar') + dest = OpCountingBytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(dest, write_size=1) as decompressor: + s = struct.Struct('>B') + for c in source: + if not isinstance(c, str): + c = s.pack(c) + decompressor.write(c) + + + self.assertEqual(dest.getvalue(), b'foobarfoobar') + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +class TestDecompressor_read_from(unittest.TestCase): + def test_type_validation(self): + dctx = zstd.ZstdDecompressor() + + # Object with read() works. + dctx.read_from(io.BytesIO()) + + # Buffer protocol works. + dctx.read_from(b'foobar') + + with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'): + dctx.read_from(True) + + def test_empty_input(self): + dctx = zstd.ZstdDecompressor() + + source = io.BytesIO() + it = dctx.read_from(source) + # TODO this is arguably wrong. Should get an error about missing frame foo. + with self.assertRaises(StopIteration): + next(it) + + it = dctx.read_from(b'') + with self.assertRaises(StopIteration): + next(it) + + def test_invalid_input(self): + dctx = zstd.ZstdDecompressor() + + source = io.BytesIO(b'foobar') + it = dctx.read_from(source) + with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'): + next(it) + + it = dctx.read_from(b'foobar') + with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'): + next(it) + + def test_empty_roundtrip(self): + cctx = zstd.ZstdCompressor(level=1, write_content_size=False) + empty = cctx.compress(b'') + + source = io.BytesIO(empty) + source.seek(0) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(source) + + # No chunks should be emitted since there is no data. + with self.assertRaises(StopIteration): + next(it) + + # Again for good measure. + with self.assertRaises(StopIteration): + next(it) + + def test_skip_bytes_too_large(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(ValueError, 'skip_bytes must be smaller than read_size'): + dctx.read_from(b'', skip_bytes=1, read_size=1) + + with self.assertRaisesRegexp(ValueError, 'skip_bytes larger than first input chunk'): + b''.join(dctx.read_from(b'foobar', skip_bytes=10)) + + def test_skip_bytes(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + output = b''.join(dctx.read_from(b'hdr' + compressed, skip_bytes=3)) + self.assertEqual(output, b'foobar') + + def test_large_output(self): + source = io.BytesIO() + source.write(b'f' * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE) + source.write(b'o') + source.seek(0) + + cctx = zstd.ZstdCompressor(level=1) + compressed = io.BytesIO(cctx.compress(source.getvalue())) + compressed.seek(0) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(compressed) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(decompressed, source.getvalue()) + + # And again with buffer protocol. + it = dctx.read_from(compressed.getvalue()) + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(decompressed, source.getvalue()) + + def test_large_input(self): + bytes = list(struct.Struct('>B').pack(i) for i in range(256)) + compressed = io.BytesIO() + input_size = 0 + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(compressed) as compressor: + while True: + compressor.write(random.choice(bytes)) + input_size += 1 + + have_compressed = len(compressed.getvalue()) > zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE + have_raw = input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2 + if have_compressed and have_raw: + break + + compressed.seek(0) + self.assertGreater(len(compressed.getvalue()), + zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(compressed) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(len(decompressed), input_size) + + # And again with buffer protocol. + it = dctx.read_from(compressed.getvalue()) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(len(decompressed), input_size) + + def test_interesting(self): + # Found this edge case via fuzzing. + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + + compressed = io.BytesIO() + with cctx.write_to(compressed) as compressor: + for i in range(256): + chunk = b'\0' * 1024 + compressor.write(chunk) + source.write(chunk) + + dctx = zstd.ZstdDecompressor() + + simple = dctx.decompress(compressed.getvalue(), + max_output_size=len(source.getvalue())) + self.assertEqual(simple, source.getvalue()) + + compressed.seek(0) + streamed = b''.join(dctx.read_from(compressed)) + self.assertEqual(streamed, source.getvalue()) + + def test_read_write_size(self): + source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b'foobarfoobar')) + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(source, read_size=1, write_size=1): + self.assertEqual(len(chunk), 1) + + self.assertEqual(source._read_count, len(source.getvalue()))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_estimate_sizes.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,17 @@ +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + + +class TestSizes(unittest.TestCase): + def test_decompression_size(self): + size = zstd.estimate_decompression_context_size() + self.assertGreater(size, 100000) + + def test_compression_size(self): + params = zstd.get_compression_parameters(3) + size = zstd.estimate_compression_context_size(params) + self.assertGreater(size, 100000)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_module_attributes.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,48 @@ +from __future__ import unicode_literals + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +class TestModuleAttributes(unittest.TestCase): + def test_version(self): + self.assertEqual(zstd.ZSTD_VERSION, (1, 1, 1)) + + def test_constants(self): + self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22) + self.assertEqual(zstd.FRAME_HEADER, b'\x28\xb5\x2f\xfd') + + def test_hasattr(self): + attrs = ( + 'COMPRESSION_RECOMMENDED_INPUT_SIZE', + 'COMPRESSION_RECOMMENDED_OUTPUT_SIZE', + 'DECOMPRESSION_RECOMMENDED_INPUT_SIZE', + 'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE', + 'MAGIC_NUMBER', + 'WINDOWLOG_MIN', + 'WINDOWLOG_MAX', + 'CHAINLOG_MIN', + 'CHAINLOG_MAX', + 'HASHLOG_MIN', + 'HASHLOG_MAX', + 'HASHLOG3_MAX', + 'SEARCHLOG_MIN', + 'SEARCHLOG_MAX', + 'SEARCHLENGTH_MIN', + 'SEARCHLENGTH_MAX', + 'TARGETLENGTH_MIN', + 'TARGETLENGTH_MAX', + 'STRATEGY_FAST', + 'STRATEGY_DFAST', + 'STRATEGY_GREEDY', + 'STRATEGY_LAZY', + 'STRATEGY_LAZY2', + 'STRATEGY_BTLAZY2', + 'STRATEGY_BTOPT', + ) + + for a in attrs: + self.assertTrue(hasattr(zstd, a))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_roundtrip.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,64 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +try: + import hypothesis + import hypothesis.strategies as strategies +except ImportError: + raise unittest.SkipTest('hypothesis not available') + +import zstd + + +compression_levels = strategies.integers(min_value=1, max_value=22) + + +class TestRoundTrip(unittest.TestCase): + @hypothesis.given(strategies.binary(), compression_levels) + def test_compress_write_to(self, data, level): + """Random data from compress() roundtrips via write_to.""" + cctx = zstd.ZstdCompressor(level=level) + compressed = cctx.compress(data) + + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed) + + self.assertEqual(buffer.getvalue(), data) + + @hypothesis.given(strategies.binary(), compression_levels) + def test_compressor_write_to_decompressor_write_to(self, data, level): + """Random data from compressor write_to roundtrips via write_to.""" + compress_buffer = io.BytesIO() + decompressed_buffer = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=level) + with cctx.write_to(compress_buffer) as compressor: + compressor.write(data) + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(decompressed_buffer) as decompressor: + decompressor.write(compress_buffer.getvalue()) + + self.assertEqual(decompressed_buffer.getvalue(), data) + + @hypothesis.given(strategies.binary(average_size=1048576)) + @hypothesis.settings(perform_health_check=False) + def test_compressor_write_to_decompressor_write_to_larger(self, data): + compress_buffer = io.BytesIO() + decompressed_buffer = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=5) + with cctx.write_to(compress_buffer) as compressor: + compressor.write(data) + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(decompressed_buffer) as decompressor: + decompressor.write(compress_buffer.getvalue()) + + self.assertEqual(decompressed_buffer.getvalue(), data)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/tests/test_train_dictionary.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,46 @@ +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + + +if sys.version_info[0] >= 3: + int_type = int +else: + int_type = long + + +class TestTrainDictionary(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + zstd.train_dictionary() + + def test_bad_args(self): + with self.assertRaises(TypeError): + zstd.train_dictionary(8192, u'foo') + + with self.assertRaises(ValueError): + zstd.train_dictionary(8192, [u'foo']) + + def test_basic(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + samples.append(b'baz' * 64) + samples.append(b'foobaz' * 64) + samples.append(b'bazfoo' * 64) + + d = zstd.train_dictionary(8192, samples) + self.assertLessEqual(len(d), 8192) + + dict_id = d.dict_id() + self.assertIsInstance(dict_id, int_type) + + data = d.as_bytes() + self.assertEqual(data[0:4], b'\x37\xa4\x30\xec')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,112 @@ +/** + * Copyright (c) 2016-present, Gregory Szorc + * All rights reserved. + * + * This software may be modified and distributed under the terms + * of the BSD license. See the LICENSE file for details. + */ + +/* A Python C extension for Zstandard. */ + +#include "python-zstandard.h" + +PyObject *ZstdError; + +PyDoc_STRVAR(estimate_compression_context_size__doc__, +"estimate_compression_context_size(compression_parameters)\n" +"\n" +"Give the amount of memory allocated for a compression context given a\n" +"CompressionParameters instance"); + +PyDoc_STRVAR(estimate_decompression_context_size__doc__, +"estimate_decompression_context_size()\n" +"\n" +"Estimate the amount of memory allocated to a decompression context.\n" +); + +static PyObject* estimate_decompression_context_size(PyObject* self) { + return PyLong_FromSize_t(ZSTD_estimateDCtxSize()); +} + +PyDoc_STRVAR(get_compression_parameters__doc__, +"get_compression_parameters(compression_level[, source_size[, dict_size]])\n" +"\n" +"Obtains a ``CompressionParameters`` instance from a compression level and\n" +"optional input size and dictionary size"); + +PyDoc_STRVAR(train_dictionary__doc__, +"train_dictionary(dict_size, samples)\n" +"\n" +"Train a dictionary from sample data.\n" +"\n" +"A compression dictionary of size ``dict_size`` will be created from the\n" +"iterable of samples provided by ``samples``.\n" +"\n" +"The raw dictionary content will be returned\n"); + +static char zstd_doc[] = "Interface to zstandard"; + +static PyMethodDef zstd_methods[] = { + { "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size, + METH_VARARGS, estimate_compression_context_size__doc__ }, + { "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size, + METH_NOARGS, estimate_decompression_context_size__doc__ }, + { "get_compression_parameters", (PyCFunction)get_compression_parameters, + METH_VARARGS, get_compression_parameters__doc__ }, + { "train_dictionary", (PyCFunction)train_dictionary, + METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ }, + { NULL, NULL } +}; + +void compressobj_module_init(PyObject* mod); +void compressor_module_init(PyObject* mod); +void compressionparams_module_init(PyObject* mod); +void constants_module_init(PyObject* mod); +void dictparams_module_init(PyObject* mod); +void compressiondict_module_init(PyObject* mod); +void compressionwriter_module_init(PyObject* mod); +void compressoriterator_module_init(PyObject* mod); +void decompressor_module_init(PyObject* mod); +void decompressobj_module_init(PyObject* mod); +void decompressionwriter_module_init(PyObject* mod); +void decompressoriterator_module_init(PyObject* mod); + +void zstd_module_init(PyObject* m) { + compressionparams_module_init(m); + dictparams_module_init(m); + compressiondict_module_init(m); + compressobj_module_init(m); + compressor_module_init(m); + compressionwriter_module_init(m); + compressoriterator_module_init(m); + constants_module_init(m); + decompressor_module_init(m); + decompressobj_module_init(m); + decompressionwriter_module_init(m); + decompressoriterator_module_init(m); +} + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef zstd_module = { + PyModuleDef_HEAD_INIT, + "zstd", + zstd_doc, + -1, + zstd_methods +}; + +PyMODINIT_FUNC PyInit_zstd(void) { + PyObject *m = PyModule_Create(&zstd_module); + if (m) { + zstd_module_init(m); + } + return m; +} +#else +PyMODINIT_FUNC initzstd(void) { + PyObject *m = Py_InitModule3("zstd", zstd_methods, zstd_doc); + if (m) { + zstd_module_init(m); + } +} +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/LICENSE Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/PATENTS Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,33 @@ +Additional Grant of Patent Rights Version 2 + +"Software" means the Zstandard software distributed by Facebook, Inc. + +Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software +("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable +(subject to the termination provision below) license under any Necessary +Claims, to make, have made, use, sell, offer to sell, import, and otherwise +transfer the Software. For avoidance of doubt, no license is granted under +Facebook’s rights in any patent claims that are infringed by (i) modifications +to the Software made by you or any third party or (ii) the Software in +combination with any software or other technology. + +The license granted hereunder will terminate, automatically and without notice, +if you (or any of your subsidiaries, corporate affiliates or agents) initiate +directly or indirectly, or take a direct financial interest in, any Patent +Assertion: (i) against Facebook or any of its subsidiaries or corporate +affiliates, (ii) against any party if such Patent Assertion arises in whole or +in part from any software, technology, product or service of Facebook or any of +its subsidiaries or corporate affiliates, or (iii) against any party relating +to the Software. Notwithstanding the foregoing, if Facebook or any of its +subsidiaries or corporate affiliates files a lawsuit alleging patent +infringement against you in the first instance, and you respond by filing a +patent infringement counterclaim in that lawsuit against that party that is +unrelated to the Software, the license granted hereunder will not terminate +under section (i) of this paragraph due to such counterclaim. + +A "Necessary Claim" is a claim of a patent owned by Facebook that is +necessarily infringed by the Software standing alone. + +A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, +or contributory infringement or inducement to infringe any patent, including a +cross-claim or counterclaim.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/bitstream.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,414 @@ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include <immintrin.h> /* support for bextr (experimental) */ +#endif + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. +* A critical property of these streams is that they encode and decode in **reverse** direction. +* So the first bit sequence you add will be the last to be read, like a LIFO stack. +*/ +typedef struct +{ + size_t bitContainer; + int bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ + + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(void*) + * @return : 0 if success, + otherwise an error code (can be tested using ERR_isError() ) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); + if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + can add up to 26 bits into `bitC`. + Does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_flushBits() : + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + + if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ + + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : +* Initialize a BIT_DStream_t. +* `bitD` : a pointer to an already allocated BIT_DStream_t structure. +* `srcSize` must be the *exact* size of the bitStream, in bytes. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + default:; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + return (bitContainer >> start) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted + */ + MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +#endif +} + +/*! BIT_lookBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : +* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). +* This function is safe, it guarantees it will not read beyond src buffer. +* @return : status of `BIT_DStream_t` internal register. + if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BIT_endOfDStream() : +* @return Tells if DStream has exactly reached its end (all bits consumed). +*/ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/entropy_common.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,225 @@ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } + +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } + +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<<nbBits)+1; + threshold = 1<<nbBits; + nbBits++; + + while ((remaining>1) & (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend-5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(corruption_detected); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n<oSize; n+=2) { + huffWeight[n] = ip[n/2] >> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n<oSize; n++) { + if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/error_private.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/error_private.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include <stddef.h> /* size_t */ +#include "zstd_errors.h" /* enum list */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/fse.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,634 @@ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef FSE_H +#define FSE_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-***************************************** +* Dependencies +******************************************/ +#include <stddef.h> /* size_t, ptrdiff_t */ + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_count(): + Provides the precise count of each byte within a table 'count'. + 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + *maxSymbolValuePtr will be updated if detected smaller than initial value. + @return : the count of the most frequent symbol (which is not identified). + if return == srcSize, there is only one symbol. + Can also return an error code, which can be tested with FSE_isError(). */ +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); +void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_DTable* FSE_createDTable(unsigned tableLog); +void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + + +#ifdef FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) + + +/* ***************************************** +* FSE advanced API +*******************************************/ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); +/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); +/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ + +size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); +/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ + + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary. + Visual seems to do it automatically. + For gcc or clang, you'll need to add -flto flag at compilation and linking stages. + If none of these solutions is applicable, include "fse.c" directly. +*/ +typedef struct +{ + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct +{ + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<<tableLog; + statePtr->stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +{ + const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG) +#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1) +#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2) +#define FSE_MIN_TABLELOG 5 + +#define FSE_TABLELOG_ABSOLUTE_MAX 15 +#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* FSE_H */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/fse_decompress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,329 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + for (i=0; i<normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u<tableSize; u++) { + FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); + U16 nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); + tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + } } + + return 0; +} + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + + /* Init */ + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) { + op[0] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) return NCountLength; + if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += NCountLength; + cSrcSize -= NCountLength; + } + + CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); + + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ +} + + + +#endif /* FSE_COMMONDEFS_ONLY */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/huf.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,228 @@ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* *** Dependencies *** */ +#include <stddef.h> /* size_t */ + + +/* *** simple functions *** */ +/** +HUF_compress() : + Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + @return : size of compressed data (<= `dstCapacity`). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single repeated byte symbol (RLE compression). + if HUF_isError(return), compression failed (more details using HUF_getErrorName()) +*/ +size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** +HUF_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUF_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== dstSize), + or an error code, which can be tested using HUF_isError() +*/ +size_t HUF_decompress(void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize); + + +/* **************************************** +* Tool functions +******************************************/ +#define HUF_BLOCKSIZE_MAX (128 * 1024) +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : +* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */ +size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +#ifdef HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ + + +/* *** Constants *** */ +#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[maxSymbolValue+1]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } +#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/** HUF_readCTable() : +* Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); + + +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +#endif /* HUF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUF_H_298734234 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/mem.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,370 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <string.h> /* memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include <stdlib.h> /* _byteswap_ulong */ +# include <intrin.h> /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/xxhash.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,867 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include <stdlib.h> +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include <string.h> +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************* +* Constants +***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ************************** +* Utils +****************************/ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + + +/* *************************** +* Simple Hash Functions +*****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +/* ************************************************** +* Advanced Hash Functions +****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + + +/* **** XXH64 **** */ + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/* ************************** +* Canonical representation +****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/xxhash.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,309 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */ +#endif + + +/* **************************** +* Definitions +******************************/ +#include <stddef.h> /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/** XXH_PRIVATE_API +* This is useful if you want to include xxhash functions in `static` mode +* in order to inline them, and remove their symbol from the public list. +* Methodology : +* #define XXH_PRIVATE_API +* #include "xxhash.h" +* `xxhash.c` is automatically included. +* It's not useful to compile and link it as a separate module anymore. +*/ +#ifdef XXH_PRIVATE_API +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Simple Hash Functions +******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + + +/* **************************** +* Streaming Hash Functions +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! State allocation, compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + + +/* ************************** +* Utils +****************************/ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ +# define restrict /* disable restrict */ +#endif + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); + + +/* ************************** +* Canonical representation +****************************/ +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ + +/* These definitions are only meant to allow allocation of XXH state + statically, on stack, or in a struct for example. + Do not use members directly. */ + + struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH32_state_t */ + + struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH64_state_t */ + + +# ifdef XXH_PRIVATE_API +# include "xxhash.c" /* include xxhash functions as `static`, for inlining */ +# endif + +#endif /* XXH_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/zbuff.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,191 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* *************************************************************** +* NOTES/WARNINGS +*****************************************************************/ +/* The streaming API defined here will soon be deprecated by the +* new one in 'zstd.h'; consider migrating towards newer streaming +* API. See 'lib/README.md'. +*****************************************************************/ + +#ifndef ZSTD_BUFFERED_H_23987 +#define ZSTD_BUFFERED_H_23987 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Dependencies +***************************************/ +#include <stddef.h> /* size_t */ + + +/* *************************************************************** +* Compiler specifics +*****************************************************************/ +/* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL */ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + +/* ************************************* +* Streaming functions +***************************************/ +/* This is the easier "buffered" streaming API, +* using an internal buffer to lift all restrictions on user-provided buffers +* which can be any size, any place, for both input and output. +* ZBUFF and ZSTD are 100% interoperable, +* frames created by one can be decoded by the other one */ + +typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); + +ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); + +ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); + +/*-************************************************* +* Streaming compression - howto +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Start by initializing ZBUF_CCtx. +* Use ZBUFF_compressInit() to start a new compression operation. +* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary. +* +* Use ZBUFF_compressContinue() repetitively to consume input stream. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). +* The nb of bytes written into `dst` will be reported into *dstCapacityPtr. +* Note that the function cannot output more than *dstCapacityPtr, +* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* In which case, call again ZBUFF_compressFlush() to complete the flush. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize() +* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency) +* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. +* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering. +* **************************************************/ + + +typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); + +ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); + +ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFF_DCtx object is required to track streaming operations. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); +ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); + +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are just hints, they tend to offer better latency */ +ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); + + +#ifdef ZBUFF_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Dependency ---*/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */ +#include "zstd.h" + + +/*--- Custom memory allocator ---*/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); + + +/*--- Advanced Streaming Initialization ---*/ +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + +#endif /* ZBUFF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_BUFFERED_H_23987 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/zstd_common.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + + +/*-************************************* +* Dependencies +***************************************/ +#include <stdlib.h> /* malloc */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ +#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTD_isError() : +* tells if a return value is an error code */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : +* convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : +* provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* ZBUFF Error Management +****************************************************************/ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + + +/*=************************************************************** +* Custom allocator +****************************************************************/ +/* default uses stdlib */ +void* ZSTD_defaultAllocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + return address; +} + +void ZSTD_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + free(address); +} + +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +{ + return customMem.customAlloc(customMem.opaque, size); +} + +void ZSTD_free(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) + customMem.customFree(customMem.opaque, ptr); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/zstd_errors.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include <stddef.h> /* size_t */ + + +/*-**************************************** +* error codes list +******************************************/ +typedef enum { + ZSTD_error_no_error, + ZSTD_error_GENERIC, + ZSTD_error_prefix_unknown, + ZSTD_error_version_unsupported, + ZSTD_error_parameter_unknown, + ZSTD_error_frameParameter_unsupported, + ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, + ZSTD_error_compressionParameter_unsupported, + ZSTD_error_init_missing, + ZSTD_error_memory_allocation, + ZSTD_error_stage_wrong, + ZSTD_error_dstSize_tooSmall, + ZSTD_error_srcSize_wrong, + ZSTD_error_corruption_detected, + ZSTD_error_checksum_wrong, + ZSTD_error_tableLog_tooLarge, + ZSTD_error_maxSymbolValue_tooLarge, + ZSTD_error_maxSymbolValue_tooSmall, + ZSTD_error_dictionary_corrupted, + ZSTD_error_dictionary_wrong, + ZSTD_error_maxCode +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +const char* ZSTD_getErrorString(ZSTD_ErrorCode code); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/common/zstd_internal.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,267 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# ifdef __GNUC__ +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + + +/*-************************************* +* shared macros +***************************************/ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ +#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) +#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) +static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 +#define EQUAL_READ32 4 + +#define Litbits 8 +#define MaxLit ((1<<Litbits) - 1) +#define MaxML 52 +#define MaxLL 35 +#define MaxOff 28 +#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ +#define MLFSELog 9 +#define LLFSELog 9 +#define OffFSELog 8 + +static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16 }; +static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 }; +#define LL_DEFAULTNORMLOG 6 /* for static allocation */ +static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; + +static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 }; +static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 }; +#define ML_DEFAULTNORMLOG 6 /* for static allocation */ +static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; + +static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; +#define OF_DEFAULTNORMLOG 5 /* for static allocation */ +static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; + + +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } +#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } + +/*! ZSTD_wildcopy() : +* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ +#define WILDCOPY_OVERLENGTH 8 +MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + do + COPY8(op, ip) + while (op < oend); +} + +MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = (BYTE*)dstEnd; + do + COPY8(op, ip) + while (op < oend); +} + + +/*-******************************************* +* Private interfaces +*********************************************/ +typedef struct ZSTD_stats_s ZSTD_stats_t; + +typedef struct { + U32 off; + U32 len; +} ZSTD_match_t; + +typedef struct { + U32 price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + + +typedef struct seqDef_s { + U32 offset; + U16 litLength; + U16 matchLength; +} seqDef; + + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; + BYTE* litStart; + BYTE* lit; + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ + U32 longLengthPos; + /* opt */ + ZSTD_optimal_t* priceTable; + ZSTD_match_t* matchTable; + U32* matchLengthFreq; + U32* litLengthFreq; + U32* litFreq; + U32* offCodeFreq; + U32 matchLengthSum; + U32 matchSum; + U32 litLengthSum; + U32 litSum; + U32 offCodeSum; + U32 log2matchLengthSum; + U32 log2matchSum; + U32 log2litLengthSum; + U32 log2litSum; + U32 log2offCodeSum; + U32 factor; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; +} seqStore_t; + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); + +/* custom memory allocation functions */ +void* ZSTD_defaultAllocFunction(void* opaque, size_t size); +void ZSTD_defaultFreeFunction(void* opaque, void* address); +static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); +void ZSTD_free(void* ptr, ZSTD_customMem customMem); + + +/*====== common function ======*/ + +MEM_STATIC U32 ZSTD_highbit32(U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse(&r, val); + return (unsigned)r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +#endif /* ZSTD_CCOMMON_H_MODULE */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/compress/fse_compress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,810 @@ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + U32 const tableSize = 1 << tableLog; + U32 const tableMask = tableSize - 1; + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ + U32 highThreshold = tableSize-1; + + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u<=maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurences; + for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) { + tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ + } } + + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u<tableSize; u++) { + FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ + tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */ + } } + + /* Build Symbol Transformation Table */ + { unsigned total = 0; + unsigned s; + for (s=0; s<=maxSymbolValue; s++) { + switch (normalizedCounter[s]) + { + case 0: break; + + case -1: + case 1: + symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog); + symbolTT[s].deltaFindState = total - 1; + total ++; + break; + default : + { + U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1); + U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = total - normalizedCounter[s]; + total += normalizedCounter[s]; + } } } } + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } + +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + bitStream = 0; + bitCount = 0; + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while (remaining>1) { /* stops at 1 */ + if (previous0) { + unsigned start = charnum; + while (!normalizedCounter[charnum]) charnum++; + while (charnum >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (charnum >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (charnum-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { short count = normalizedCounter[charnum++]; + const short max = (short)((2*threshold-1)-remaining); + remaining -= FSE_abs(count); + if (remaining<1) return ERROR(GENERIC); + count++; /* +1 for extra accuracy */ + if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count<max); + previous0 = (count==1); + while (remaining<threshold) nbBits--, threshold>>=1; + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); +} + + + +/*-************************************************************** +* Counting histogram +****************************************************************/ +/*! FSE_count_simple + This function just counts byte values within `src`, + and store the histogram into table `count`. + This function is unsafe : it doesn't check that all values within `src` can fit into `count`. + For this reason, prefer using a table `count` with 256 elements. + @return : count of most numerous element +*/ +static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip<end) count[*ip++]++; + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + + { U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; } + + return (size_t)max; +} + + +static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + unsigned checkMax) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + + U32 Counting1[256] = { 0 }; + U32 Counting2[256] = { 0 }; + U32 Counting3[256] = { 0 }; + U32 Counting4[256] = { 0 }; + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip<iend) Counting1[*ip++]++; + + if (checkMax) { /* verify stats will fit into destination table */ + U32 s; for (s=255; s>maxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); + } } + + { U32 s; for (s=0; s<=maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) max = count[s]; + }} + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); + return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0); +} + +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + if (*maxSymbolValuePtr <255) + return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1); + *maxSymbolValuePtr = 255; + return FSE_countFast(count, maxSymbolValuePtr, source, sourceSize); +} + + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ +/*! FSE_sizeof_CTable() : + FSE_CTable is a variable size structure which contains : + `U16 tableLog;` + `U16 maxSymbolValue;` + `U16 nextStateNumber[1 << tableLog];` // This size is variable + `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable +Allocation is manual (C standard does not support variable-size structures). +*/ + +size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return size; +} + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + norm[s]=-2; + } + ToDistribute = (1 << tableLog) - distributed; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == -2) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) maxV=s, maxC=count[s]; + norm[maxV] += (short)ToDistribute; + return 0; + } + + { + U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ + U64 tmpTotal = mid; + for (s=0; s<=maxSymbolValue; s++) { + if (norm[s]==-2) { + U64 end = tmpTotal + (count[s] * rStep); + U32 sStart = (U32)(tmpTotal >> vStepLog); + U32 sEnd = (U32)(end >> vStepLog); + U32 weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<<tableLog; + unsigned s; + unsigned largest=0; + short largestP=0; + U32 lowThreshold = (U32)(total >> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat; + } + if (proba > largestP) largestP=proba, largest=s; + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + printf("%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<<tableLog)) + printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog); + getchar(); + } +#endif + + return tableLog; +} + + +/* fake FSE_CTable, for raw (uncompressed) input */ +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) +{ + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSymbolValue = tableMask; + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s<tableSize; s++) + tableU16[s] = (U16)(tableSize + s); + + /* Build Symbol Transformation Table */ + { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); + + for (s=0; s<=maxSymbolValue; s++) { + symbolTT[s].deltaNbBits = deltaNbBits; + symbolTT[s].deltaFindState = s-1; + } } + + + return 0; +} + +/* fake FSE_CTable, for rle (100% always same symbol) input */ +size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) +{ + void* ptr = ct; + U16* tableU16 = ( (U16*) ptr) + 2; + void* FSCTptr = (U32*)ptr + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; + + /* header */ + tableU16[-2] = (U16) 0; + tableU16[-1] = (U16) symbolValue; + + /* Build table */ + tableU16[0] = 0; + tableU16[1] = 0; /* just in case */ + + /* Build Symbol Transformation Table */ + symbolTT[symbolValue].deltaNbBits = 0; + symbolTT[symbolValue].deltaFindState = 0; + + return 0; +} + + +static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct, const unsigned fast) +{ + const BYTE* const istart = (const BYTE*) src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip=iend; + + + BIT_CStream_t bitC; + FSE_CState_t CState1, CState2; + + /* init */ + if (srcSize <= 2) return 0; + { size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize); + if (FSE_isError(errorCode)) return 0; } + +#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + + if (srcSize & 1) { + FSE_initCState2(&CState1, ct, *--ip); + FSE_initCState2(&CState2, ct, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } else { + FSE_initCState2(&CState2, ct, *--ip); + FSE_initCState2(&CState1, ct, *--ip); + } + + /* join to mod 4 */ + srcSize -= 2; + if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + for ( ; ip>istart ; ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + const BYTE* const istart = (const BYTE*) src; + const BYTE* ip = istart; + + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + U32 count[FSE_MAX_SYMBOL_VALUE+1]; + S16 norm[FSE_MAX_SYMBOL_VALUE+1]; + CTable_max_t ct; + size_t errorCode; + + /* init conditions */ + if (srcSize <= 1) return 0; /* Uncompressible */ + if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; + + /* Scan input and build symbol stats */ + errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode == srcSize) return 1; + if (errorCode == 1) return 0; /* each symbol only present once */ + if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + + /* Write table description header */ + errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + op += errorCode; + + /* Compress */ + errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct); + if (errorCode == 0) return 0; /* not enough space for compressed data */ + op += errorCode; + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) + return 0; + + return op-ostart; +} + +size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + + +#endif /* FSE_COMMONDEFS_ONLY */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/compress/huf_compress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,533 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within "huf.h" */ + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +/*! HUF_writeCTable() : + `CTable` : huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; + BYTE* op = (BYTE*)dst; + U32 n; + + /* check conditions */ + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n<huffLog+1; n++) + bitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; n<maxSymbolValue; n++) + huffWeight[n] = bitsToWeight[CTable[n].nbBits]; + + { size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); + if (FSE_isError(size)) return size; + if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)size; + return size+1; + } + } + + /* raw values */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ + for (n=0; n<maxSymbolValue; n+=2) + op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; + +} + + +size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + size_t readSize; + U32 nbSymbols = 0; + /*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */ + + /* get symbol weights */ + readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(readSize)) return readSize; + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; n<nbSymbols; n++) { + const U32 w = huffWeight[n]; + CTable[n].nbBits = (BYTE)(tableLog + 1 - w); + } } + + /* fill val */ + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } + /* determine stating value per rank */ + valPerRank[tableLog+1] = 0; /* for w==0 */ + { U16 min = 0; + U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } + } + + return readSize; +} + + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1]; + nodeElt* huffNode = huffNode0 + 1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNode0)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(errorCode)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + case 0 : + default: ; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; + op += cSize; + } + + return op-ostart; +} + + +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + U32 count[HUF_SYMBOLVALUE_MAX+1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1]; + + /* checks & inits */ + if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ + if (!dstSize) return 0; /* cannot fit within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Scan input and build symbol stats */ + { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); + if (HUF_isError(largest)) return largest; + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); + if (HUF_isError(maxBits)) return maxBits; + huffLog = (U32)maxBits; + } + + /* Write table description header */ + { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); + if (HUF_isError(hSize)) return hSize; + if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + op += hSize; + } + + /* Compress */ + { size_t const cSize = (singleStream) ? + HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */ + HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; /* uncompressible */ + op += cSize; + } + + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) + return 0; + + return op-ostart; +} + + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); +} + + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/compress/zbuff_compress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,319 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#include <stdlib.h> +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" + + +/* ************************************* +* Constants +***************************************/ +static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; + + +/*-*********************************************************** +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *dstCapacityPtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. +* ***********************************************************/ + +typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage; + +/* *** Resources *** */ +struct ZBUFF_CCtx_s { + ZSTD_CCtx* zc; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZBUFF_cStage stage; + U32 checksum; + U32 frameEnded; + ZSTD_customMem customMem; +}; /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */ + +ZBUFF_CCtx* ZBUFF_createCCtx(void) +{ + return ZBUFF_createCCtx_advanced(defaultCustomMem); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_CCtx* zbc; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx)); + if (zbc==NULL) return NULL; + memset(zbc, 0, sizeof(ZBUFF_CCtx)); + memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); + zbc->zc = ZSTD_createCCtx_advanced(customMem); + if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; } + return zbc; +} + +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) +{ + if (zbc==NULL) return 0; /* support free on NULL */ + ZSTD_freeCCtx(zbc->zc); + if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); + if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); + zbc->customMem.customFree(zbc->customMem.opaque, zbc); + return 0; +} + + +/* ====== Initialization ====== */ + +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zbc->inBuffSize < neededInBuffSize) { + zbc->inBuffSize = neededInBuffSize; + zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); + } + if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { + zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; + zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } + + { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } + + zbc->inToCompress = 0; + zbc->inBuffPos = 0; + zbc->inBuffTarget = zbc->blockSize; + zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_load; + zbc->checksum = params.fParams.checksumFlag > 0; + zbc->frameEnded = 0; + return 0; /* ready to go */ +} + + +size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); +} + +size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) +{ + return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel); +} + + +/* internal util function */ +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +/* ====== Compression ====== */ + +typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e; + +static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + ZBUFF_flush_e const flush) +{ + U32 someMoreWork = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (someMoreWork) { + switch(zbc->stage) + { + case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case ZBUFFcs_load: + /* complete inBuffer */ + { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; + size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); + zbc->inBuffPos += loaded; + ip += loaded; + if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { + someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zbc->inBuffPos - zbc->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zbc->outBuff, oSize = zbc->outBuffSize; + cSize = (flush == zbf_end) ? + ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) : + ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + if (flush == zbf_end) zbc->frameEnded = 1; + /* prepare next block */ + zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; + if (zbc->inBuffTarget > zbc->inBuffSize) + zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ + zbc->inToCompress = zbc->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zbc->outBuffContentSize = cSize; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ + } + + case ZBUFFcs_flush: + { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + op += flushed; + zbc->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_load; + break; + } + + case ZBUFFcs_final: + someMoreWork = 0; /* do nothing */ + break; + + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + if (zbc->frameEnded) return 0; + { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; + if (hintInSize==0) hintInSize = zbc->blockSize; + return hintInSize; + } +} + +size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather); +} + + + +/* ====== Finalize ====== */ + +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t srcSize = 0; + ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */ + return zbc->outBuffContentSize - zbc->outBuffFlushedSize; +} + + +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + *dstCapacityPtr; + BYTE* op = ostart; + + if (zbc->stage != ZBUFFcs_final) { + /* flush whatever remains */ + size_t outSize = *dstCapacityPtr; + size_t srcSize = 0; + size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */ + size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + op += outSize; + if (remainingToFlush) { + *dstCapacityPtr = op-ostart; + return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4); + } + /* create epilogue */ + zbc->stage = ZBUFFcs_final; + zbc->outBuffContentSize = !notEnded ? 0 : + ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + op += flushed; + zbc->outBuffFlushedSize += flushed; + *dstCapacityPtr = op-ostart; + if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */ + return toFlush - flushed; + } +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/compress/zstd_compress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,3264 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/*-************************************* +* Dependencies +***************************************/ +#include <string.h> /* memset */ +#include "mem.h" +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH_reset, update, digest */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" /* includes zstd.h */ + + +/*-************************************* +* Constants +***************************************/ +static const U32 g_searchStrength = 8; /* control skip over incompressible data */ +#define HASH_READ_SIZE 8 +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; + + +/*-************************************* +* Helper functions +***************************************/ +size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } + + +/*-************************************* +* Sequence storage +***************************************/ +static void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CCtx_s +{ + const BYTE* nextSrc; /* next block here to continue on current prefix */ + const BYTE* base; /* All regular indexes relative to this position */ + const BYTE* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more data */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 nextToUpdate3; /* index from which to continue dictionary update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 loadedDictEnd; + ZSTD_compressionStage_e stage; + U32 rep[ZSTD_REP_NUM]; + U32 savedRep[ZSTD_REP_NUM]; + U32 dictID; + ZSTD_parameters params; + void* workSpace; + size_t workSpaceSize; + size_t blockSize; + U64 frameContentSize; + XXH64_state_t xxhState; + ZSTD_customMem customMem; + + seqStore_t seqStore; /* sequences storage ptrs */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + HUF_CElt* hufTable; + U32 flagStaticTables; + FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; +}; + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(defaultCustomMem); +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_CCtx* cctx; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + memset(cctx, 0, sizeof(ZSTD_CCtx)); + memcpy(&(cctx->customMem), &customMem, sizeof(customMem)); + return cctx; +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + ZSTD_free(cctx->workSpace, cctx->customMem); + ZSTD_free(cctx, cctx->customMem); + return 0; /* reserved as a potential error code in the future */ +} + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*cctx) + cctx->workSpaceSize; +} + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ +{ + return &(ctx->seqStore); +} + +static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) +{ + return cctx->params; +} + + +/** ZSTD_checkParams() : + ensure param values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); } + CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; + U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; + CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } + CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); + return 0; +} + + +/** ZSTD_adjustCParams() : + optimize `cPar` for a given input (`srcSize` and `dictSize`). + mostly downsizing to reduce memory consumption and initialization. + Both `srcSize` and `dictSize` are optional (use 0 if unknown), + but if both are 0, no optimization can be done. + Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ + + /* resize params, to use less memory when necessary */ + { U32 const minSrcSize = (srcSize==0) ? 500 : 0; + U64 const rSize = srcSize + dictSize + minSrcSize; + if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { + U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } } + if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2); + U32 const maxChainLog = cPar.windowLog+btPlus; + if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + + return cPar; +} + + +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) +{ + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); + U32 const divider = (cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + + size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); + size_t const hSize = ((size_t)1) << cParams.hashLog; + U32 const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); + size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); + + return sizeof(ZSTD_CCtx) + neededSpace; +} + + +static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) +{ + return (param1.cParams.hashLog == param2.cParams.hashLog) + & (param1.cParams.chainLog == param2.cParams.chainLog) + & (param1.cParams.strategy == param2.cParams.strategy) + & ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3)); +} + +/*! ZSTD_continueCCtx() : + reuse CCtx without reset (note : requires no dictionary) */ +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize) +{ + U32 const end = (U32)(cctx->nextSrc - cctx->base); + cctx->params = params; + cctx->frameContentSize = frameContentSize; + cctx->lowLimit = end; + cctx->dictLimit = end; + cctx->nextToUpdate = end+1; + cctx->stage = ZSTDcs_init; + cctx->dictID = 0; + cctx->loadedDictEnd = 0; + { int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; } + cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ + XXH64_reset(&cctx->xxhState, 0); + return 0; +} + +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; + +/*! ZSTD_resetCCtx_advanced() : + note : 'params' must be validated */ +static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, + ZSTD_parameters params, U64 frameContentSize, + ZSTD_compResetPolicy_e const crp) +{ + if (crp == ZSTDcrp_continue) + if (ZSTD_equivalentParams(params, zc->params)) + return ZSTD_continueCCtx(zc, params, frameContentSize); + + { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); + U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); + size_t const hSize = ((size_t)1) << params.cParams.hashLog; + U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + void* ptr; + + /* Check if workSpace is large enough, alloc a new one if needed */ + { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); + size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + + (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); + if (zc->workSpaceSize < neededSpace) { + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); + if (zc->workSpace == NULL) return ERROR(memory_allocation); + zc->workSpaceSize = neededSpace; + } } + + if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ + XXH64_reset(&zc->xxhState, 0); + zc->hashLog3 = hashLog3; + zc->hashTable = (U32*)(zc->workSpace); + zc->chainTable = zc->hashTable + hSize; + zc->hashTable3 = zc->chainTable + chainSize; + ptr = zc->hashTable3 + h3Size; + zc->hufTable = (HUF_CElt*)ptr; + zc->flagStaticTables = 0; + ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ + + zc->nextToUpdate = 1; + zc->nextSrc = NULL; + zc->base = NULL; + zc->dictBase = NULL; + zc->dictLimit = 0; + zc->lowLimit = 0; + zc->params = params; + zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; + { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } + + if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { + zc->seqStore.litFreq = (U32*)ptr; + zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); + zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); + zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); + ptr = zc->seqStore.offCodeFreq + (MaxOff+1); + zc->seqStore.matchTable = (ZSTD_match_t*)ptr; + ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; + zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; + ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; + zc->seqStore.litLengthSum = 0; + } + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; + zc->seqStore.llCode = (BYTE*) ptr; + zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; + zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; + zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->loadedDictEnd = 0; + + return 0; + } +} + + +/*! ZSTD_copyCCtx() : +* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. +* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); + + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset); + + /* copy tables */ + { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); + size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; + size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); + } + + /* copy dictionary offsets */ + dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; + dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; + dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->dictID = srcCCtx->dictID; + + /* copy entropy tables */ + dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; + if (srcCCtx->flagStaticTables) { + memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); + memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); + memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); + memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); + } + + return 0; +} + + +/*! ZSTD_reduceTable() : +* reduce table indexes by `reducerValue` */ +static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) +{ + U32 u; + for (u=0 ; u < size ; u++) { + if (table[u] < reducerValue) table[u] = 0; + else table[u] -= reducerValue; + } +} + +/*! ZSTD_reduceIndex() : +* rescale all indexes to avoid future overflow (indexes are U32) */ +static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +{ + { U32 const hSize = 1 << zc->params.cParams.hashLog; + ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } + + { U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); + ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } + + { U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; + ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + MEM_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); + return ZSTD_blockHeaderSize+srcSize; +} + + +static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + default: /*note : should not be necessary : flSize is within {1,2,3} */ + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + } + + ostart[flSize] = *(const BYTE*)src; + return flSize+1; +} + + +static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } + +static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t const minGain = ZSTD_minGain(srcSize); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + + /* small ? don't even attempt compression (speed opt) */ +# define LITERAL_NOENTROPY 63 + { size_t const minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + if (zc->flagStaticTables && (lhSize==3)) { + hType = set_repeat; + singleStream = 1; + cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable); + } else { + cLitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11) + : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11); + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + if (cLitSize==1) + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + default: /* should not be necessary, lhSize is only {3,4,5} */ + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + } + return lhSize+cLitSize; +} + +static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + +static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + BYTE const LL_deltaCode = 19; + BYTE const ML_deltaCode = 36; + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + for (u=0; u<nbSeq; u++) { + U32 const llv = sequences[u].litLength; + U32 const mlv = sequences[u].matchLength; + llCodeTable[u] = (llv> 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; + ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); + mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; + } + if (seqStorePtr->longLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + + +size_t ZSTD_compressSequences(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + size_t srcSize) +{ + const seqStore_t* seqStorePtr = &(zc->seqStore); + U32 count[MaxSeq+1]; + S16 norm[MaxSeq+1]; + FSE_CTable* CTable_LitLength = zc->litlengthCTable; + FSE_CTable* CTable_OffsetBits = zc->offcodeCTable; + FSE_CTable* CTable_MatchLength = zc->matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* seqHead; + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const litSize = seqStorePtr->lit - literals; + size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); + if (ZSTD_isError(cSize)) return cSize; + op += cSize; + } + + /* Sequences Header */ + if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) goto _check_compressibility; + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + +#define MIN_SEQ_FOR_DYNAMIC_FSE 64 +#define MAX_SEQ_FOR_STATIC_FSE 1000 + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + + /* CTable for Literal Lengths */ + { U32 max = MaxLL; + size_t const mostFrequent = FSE_countFast(count, &max, llCodeTable, nbSeq); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = llCodeTable[0]; + FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); + LLtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + LLtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) { + FSE_buildCTable(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog); + LLtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); + if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable(CTable_LitLength, norm, max, tableLog); + LLtype = set_compressed; + } } + + /* CTable for Offsets */ + { U32 max = MaxOff; + size_t const mostFrequent = FSE_countFast(count, &max, ofCodeTable, nbSeq); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = ofCodeTable[0]; + FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); + Offtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + Offtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { + FSE_buildCTable(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog); + Offtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); + if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog); + Offtype = set_compressed; + } } + + /* CTable for MatchLengths */ + { U32 max = MaxML; + size_t const mostFrequent = FSE_countFast(count, &max, mlCodeTable, nbSeq); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = *mlCodeTable; + FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); + MLtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + MLtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) { + FSE_buildCTable(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog); + MLtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); + if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable(CTable_MatchLength, norm, max, tableLog); + MLtype = set_compressed; + } } + + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + zc->flagStaticTables = 0; + + /* Encoding Sequences */ + { BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ + BYTE const llCode = llCodeTable[n]; + BYTE const ofCode = ofCodeTable[n]; + BYTE const mlCode = mlCodeTable[n]; + U32 const llBits = LL_bits[llCode]; + U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ + U32 const mlBits = ML_bits[mlCode]; + /* (7)*/ /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ + FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ + if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + BIT_flushBits(&blockStream); /* (7)*/ + } } + + FSE_flushCState(&blockStream, &stateMatchLength); + FSE_flushCState(&blockStream, &stateOffsetBits); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + op += streamSize; + } } + + /* check compressibility */ +_check_compressibility: + { size_t const minGain = ZSTD_minGain(srcSize); + size_t const maxCSize = srcSize - minGain; + if ((size_t)(op-ostart) >= maxCSize) return 0; } + + /* confirm repcodes */ + { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; } + + return op - ostart; +} + + +/*! ZSTD_storeSeq() : + Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. + `offsetCode` : distance to match, or 0 == repCode. + `matchCode` : matchLength - MINMATCH +*/ +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) +{ +#if 0 /* for debug */ + static const BYTE* g_start = NULL; + const U32 pos = (U32)(literals - g_start); + if (g_start==NULL) g_start = literals; + //if ((pos > 1) && (pos < 50000)) + printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", + pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); +#endif + /* copy Literals */ + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offset = offsetCode + 1; + + /* match Length */ + if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].matchLength = (U16)matchCode; + + seqStorePtr->sequences++; +} + + +/*-************************************* +* Match length counter +***************************************/ +static unsigned ZSTD_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } + if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++; + return (size_t)(pIn - pStart); +} + +/** ZSTD_count_2segments() : +* can count match length with `ip` & `match` in 2 different segments. +* convention : on reaching mEnd, match count continue starting from iStart +*/ +static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) +{ + const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); + size_t const matchLength = ZSTD_count(ip, match, vEnd); + if (match + matchLength != mEnd) return matchLength; + return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); +} + + +/*-************************************* +* Hashes +***************************************/ +static const U32 prime3bytes = 506832829U; +static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + + +/*-************************************* +* Fast Scan +***************************************/ +static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) +{ + U32* const hashTable = zc->hashTable; + U32 const hBits = zc->params.cParams.hashLog; + const BYTE* const base = zc->base; + const BYTE* ip = base + zc->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashTable = cctx->hashTable; + U32 const hBits = cctx->params.cParams.hashLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hBits, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + hashTable[h] = current; /* update hash table */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; + cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return; + } +} + + +static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* hashTable = ctx->hashTable; + const U32 hBits = ctx->params.cParams.hashLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hBits, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashTable[h] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ( (matchIndex < lowestIndex) || + (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset = current - matchIndex; + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return; + } +} + + +/*-************************************* +* Double Fast +***************************************/ +static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls) +{ + U32* const hashLarge = cctx->hashTable; + U32 const hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + U32 const hBitsS = cctx->params.cParams.chainLog; + const BYTE* const base = cctx->base; + const BYTE* ip = base + cctx->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); + hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = cctx->hashTable; + const U32 hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + const U32 hBitsS = cctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 const matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */ + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* match3 = base + matchIndex3; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, match3+8, iend) + 8; + ip++; + offset = (U32)(ip-match3); + while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; + cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = ctx->hashTable; + U32 const hBitsL = ctx->params.cParams.hashLog; + U32* const hashSmall = ctx->chainTable; + U32 const hBitsS = ctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* matchLongBase = matchLongIndex < dictLimit ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, lowPrefixPtr) + 8; + offset = current - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; + ip++; + offset = current+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + offset = current - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; + } +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. +* ip : assumed <= iend-8 . +* @return : nb of positions added */ +static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->params.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = zc->lowLimit; + U32 matchEndIdx = current+8; + size_t bestLength = 8; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + + if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + if (matchEndIdx > current + 8) return matchEndIdx - current - 8; + return 1; +} + + +static size_t ZSTD_insertBtAndFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 nbCompares, const U32 mls, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->params.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return bestLength; +} + + +static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); +} + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + +static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); +} + + +/** Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->params.cParams.hashLog; + U32* const chainTable = zc->chainTable; + const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + zc->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + + + +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_HcFindBestMatch_generic ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls, const U32 extDict) +{ + U32* const chainTable = zc->chainTable; + const U32 chainSize = (1 << zc->params.cParams.chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 lowLimit = zc->lowLimit; + const U32 current = (U32)(ip-base); + const U32 minChain = current > chainSize ? current - chainSize : 0; + int nbAttempts=maxNbAttempts; + size_t ml=EQUAL_READ32-1; + + /* HC4 match finder */ + U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); + + for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex >= dictLimit) { + match = base + matchIndex; + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; + } + + /* save best solution */ + if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + return ml; +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + } +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); + } +} + + +/* ******************************* +* Common parser - lazy strategy +*********************************/ +FORCE_INLINE +void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base + ctx->dictLimit; + + U32 const maxSearches = 1 << ctx->params.cParams.searchLog; + U32 const mls = ctx->params.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; + U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset=0; + + /* init */ + ip += (ip==base); + ctx->nextToUpdate3 = ctx->nextToUpdate; + { U32 const maxRep = (U32)(ip-base); + if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + } + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + + /* check repCode */ + if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { + /* repcode : we take it */ + matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip<ilimit) { + ip ++; + if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip<ilimit)) { + ip ++; + if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + int const gain2 = (int)(ml2 * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = ml2, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ + { start--; matchLength++; } + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ((offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } + + /* Save reps for next block */ + ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset; + ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); +} + +static void ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); +} + +static void ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); +} + +static void ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); +} + + +FORCE_INLINE +void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ctx->lowLimit; + + const U32 maxSearches = 1 << ctx->params.cParams.searchLog; + const U32 mls = ctx->params.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; + + U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ip += (ip == prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + U32 current = (U32)(ip-base); + + /* check repCode */ + { const U32 repIndex = (U32)(current+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip<ilimit) { + ip ++; + current++; + /* check repCode */ + if (offset) { + const U32 repIndex = (U32)(current - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 1 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip<ilimit)) { + ip ++; + current++; + /* check repCode */ + if (offset) { + const U32 repIndex = (U32)(current - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + int gain2 = (int)(repLength * 4); + int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 2 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repIndex = (U32)((ip-base) - offset_2); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); +} + +static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); +} + +static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); +} + +static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); +} + + +/* The optimal parser */ +#include "zstd_opt.h" + +static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + + +typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) +{ + static const ZSTD_blockCompressor blockCompressor[2][8] = { + { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, + { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } + }; + + return blockCompressor[extDict][(U32)strat]; +} + + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); + const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ + ZSTD_resetSeqStore(&(zc->seqStore)); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ + blockCompressor(zc, src, srcSize); + return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); +} + + +/*! ZSTD_compress_generic() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = 1 << cctx->params.cParams.windowLog; + + if (cctx->params.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + size_t cSize; + + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (remaining < blockSize) blockSize = remaining; + + /* preemptive overflow correction */ + if (cctx->lowLimit > (1<<30)) { + U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2); + U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1; + U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */); + U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ + U32 const correction = cctx->lowLimit - newLowLimit; + ZSTD_reduceIndex(cctx, correction); + cctx->base += correction; + cctx->dictBase += correction; + cctx->lowLimit = newLowLimit; + cctx->dictLimit -= correction; + if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; + else cctx->nextToUpdate -= correction; + } + + if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { + /* enforce maxDist */ + U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; + if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; + } + + cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); + if (ZSTD_isError(cSize)) return cSize; + + if (cSize == 0) { /* block is not compressible */ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); + if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ + memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_blockHeaderSize+blockSize; + } else { + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader24); + cSize += ZSTD_blockHeaderSize; + } + + remaining -= blockSize; + dstCapacity -= cSize; + ip += blockSize; + op += cSize; + } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return op-ostart; +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const checksumFlag = params.fParams.checksumFlag>0; + U32 const windowSize = 1U << params.cParams.windowLog; + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ + 0; + BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos; + + if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + op[4] = frameHeaderDecriptionByte; pos=5; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: /* impossible */ + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + const BYTE* const ip = (const BYTE*) src; + size_t fhSize = 0; + + if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + /* Check if blocks follow each other */ + if (src != cctx->nextSrc) { + /* not contiguous */ + ptrdiff_t const delta = cctx->nextSrc - ip; + cctx->lowLimit = cctx->dictLimit; + cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); + cctx->dictBase = cctx->base; + cctx->base -= delta; + cctx->nextToUpdate = cctx->dictLimit; + if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ + } + + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; + cctx->lowLimit = lowLimitMax; + } + + cctx->nextSrc = ip + srcSize; + + { size_t const cSize = frame ? + ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); + if (ZSTD_isError(cSize)) return cSize; + return cSize + fhSize; + } +} + + +size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); +} + + +size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx) +{ + return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); + if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); +} + + +static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + /* input becomes current prefix */ + zc->lowLimit = zc->dictLimit; + zc->dictLimit = (U32)(zc->nextSrc - zc->base); + zc->dictBase = zc->base; + zc->base += ip - zc->nextSrc; + zc->nextToUpdate = zc->dictLimit; + zc->loadedDictEnd = (U32)(iend - zc->base); + + zc->nextSrc = iend; + if (srcSize <= HASH_READ_SIZE) return 0; + + switch(zc->params.cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + ZSTD_insertAndFindFirstIndex (zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); + break; + + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btopt2: + ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); + break; + + default: + return ERROR(GENERIC); /* strategy doesn't exist; impossible */ + } + + zc->nextToUpdate = zc->loadedDictEnd; + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + when FSE encoding. Refuse dictionaries that assign zero probability to symbols + that we may encounter during compression. + NOTE: This behavior is not standard and could be improved in the future. */ +static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); + } + return 0; +} + + +/* Dictionary format : + Magic == ZSTD_DICT_MAGIC (4 bytes) + HUF_writeCTable(256) + FSE_writeNCount(off) + FSE_writeNCount(ml) + FSE_writeNCount(ll) + RepOffsets + Dictionary content +*/ +/*! ZSTD_loadDictEntropyStats() : + @return : size read from dictionary + note : magic number supposed already checked */ +static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + + { size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize); + if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + /* Every match length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + /* Every literal length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + { U32 offcodeMax = MaxOff; + if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ + /* Calculate minimum offset code required to represent maxOffset */ + offcodeMax = ZSTD_highbit32(maxOffset); + } + /* Every possible supported offset <= dictContentSize + 128 KB must be representable */ + CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + } + + cctx->flagStaticTables = 1; + return dictPtr - (const BYTE*)dict; +} + +/** ZSTD_compress_insertDictionary() : +* @return : 0, or an error code */ +static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) +{ + if ((dict==NULL) || (dictSize<=8)) return 0; + + /* default : dict is pure content */ + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); + zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); + + /* known magic number : dict is parsed for entropy stats and content */ + { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); + size_t const eSize = loadError + 8; + if (ZSTD_isError(loadError)) return loadError; + return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); + } +} + + +/*! ZSTD_compressBegin_internal() : +* @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, U64 pledgedSrcSize) +{ + ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; + CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize); +} + + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* compression parameters verification and optimization */ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); +} + + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); +} + + +size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->params.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + + +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); + if (ZSTD_isError(cSize)) return cSize; + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + if (ZSTD_isError(endResult)) return endResult; + return cSize + endResult; +} + + +static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + size_t result; + ZSTD_CCtx ctxBody; + memset(&ctxBody, 0, sizeof(ctxBody)); + memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_free(ctxBody.workSpace, defaultCustomMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ + return result; +} + + +/* ===== Dictionary API ===== */ + +struct ZSTD_CDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + + if (!dictContent || !cdict || !cctx) { + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); + return NULL; + } + + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } + { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); + if (ZSTD_isError(errorCode)) { + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); + return NULL; + } } + + cdict->dictContent = dictContent; + cdict->dictContentSize = dictSize; + cdict->refContext = cctx; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->refContext->customMem; + ZSTD_freeCCtx(cdict->refContext); + ZSTD_free(cdict->dictContent, cMem); + ZSTD_free(cdict, cMem); + return 0; + } +} + +static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { + return ZSTD_getParamsFromCCtx(cdict->refContext); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) +{ + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) + else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + return 0; +} + +/*! ZSTD_compress_usingCDict() : +* Compression using a digested Dictionary. +* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); + + if (cdict->refContext->params.fParams.contentSizeFlag==1) { + cctx->params.fParams.contentSizeFlag = 1; + cctx->frameContentSize = srcSize; + } + + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; + +struct ZSTD_CStream_s { + ZSTD_CCtx* cctx; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage stage; + U32 checksum; + U32 frameEnded; + ZSTD_parameters params; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_CStream within "zstd.h" */ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + return ZSTD_createCStream_advanced(defaultCustomMem); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_CStream* zcs; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); + if (zcs==NULL) return NULL; + memset(zcs, 0, sizeof(ZSTD_CStream)); + memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); + zcs->cctx = ZSTD_createCCtx_advanced(customMem); + if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } + return zcs; +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = zcs->customMem; + ZSTD_freeCCtx(zcs->cctx); + ZSTD_freeCDict(zcs->cdictLocal); + ZSTD_free(zcs->inBuff, cMem); + ZSTD_free(zcs->outBuff, cMem); + ZSTD_free(zcs, cMem); + return 0; + } +} + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } + +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */ + + if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) + else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zcs->inBuffSize < neededInBuffSize) { + zcs->inBuffSize = neededInBuffSize; + ZSTD_free(zcs->inBuff, zcs->customMem); + zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); + if (zcs->inBuff == NULL) return ERROR(memory_allocation); + } + zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); + } + if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { + zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + ZSTD_free(zcs->outBuff, zcs->customMem); + zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); + if (zcs->outBuff == NULL) return ERROR(memory_allocation); + } + + if (dict) { + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + zcs->cdict = zcs->cdictLocal; + } else zcs->cdict = NULL; + + zcs->checksum = params.fParams.checksumFlag > 0; + zcs->params = params; + + return ZSTD_resetCStream(zcs, pledgedSrcSize); +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); + size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); + zcs->cdict = cdict; + return initError; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support sizeof on NULL */ + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize; +} + +/*====== Compression ======*/ + +typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + ZSTD_flush_e const flush) +{ + U32 someMoreWork = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (someMoreWork) { + switch(zcs->stage) + { + case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case zcss_load: + /* complete inBuffer */ + { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { + someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = (flush == zsf_end) ? + ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + if (flush == zsf_end) zcs->frameEnded = 1; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_flush; /* pass-through to flush stage */ + } + + case zcss_flush: + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + break; + } + + case zcss_final: + someMoreWork = 0; /* do nothing */ + break; + + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + if (zcs->frameEnded) return 0; + { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; + if (hintInSize==0) hintInSize = zcs->blockSize; + return hintInSize; + } +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + size_t sizeRead = input->size - input->pos; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + (const char*)(input->src) + input->pos, &sizeRead, zsf_gather); + input->pos += sizeRead; + output->pos += sizeWritten; + return result; +} + + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : +* @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + &srcSize, &srcSize, /* use a valid src address instead of NULL */ + zsf_flush); + output->pos += sizeWritten; + if (ZSTD_isError(result)) return result; + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + BYTE* const ostart = (BYTE*)(output->dst) + output->pos; + BYTE* const oend = (BYTE*)(output->dst) + output->size; + BYTE* op = ostart; + + if (zcs->stage != zcss_final) { + /* flush whatever remains */ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ + size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + op += sizeWritten; + if (remainingToFlush) { + output->pos += sizeWritten; + return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); + } + /* create epilogue */ + zcs->stage = zcss_final; + zcs->outBuffContentSize = !notEnded ? 0 : + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + output->pos += op-ostart; + if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ + return toFlush - flushed; + } +} + + + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_DEFAULT_CLEVEL 1 +#define ZSTD_MAX_CLEVEL 22 +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" */ + /* W, C, H, S, L, TL, strat */ + { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ + { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ + { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ + { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ + { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ + { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ + { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ + { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ + { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */ + { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */ + { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ + { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ + { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ + { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ + { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ + { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/ + { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/ + { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 7, 8, ZSTD_fast }, /* level 0 - not used */ + { 17, 12, 13, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 17, 13, 16, 1, 5, 8, ZSTD_fast }, /* level 2 */ + { 17, 16, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ + { 17, 13, 15, 3, 4, 8, ZSTD_greedy }, /* level 4 */ + { 17, 15, 17, 4, 4, 8, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ + { 17, 15, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ + { 17, 17, 17, 8, 4, 8, ZSTD_lazy2 }, /* level 12 */ + { 17, 18, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13.*/ + { 17, 17, 17, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ + { 17, 17, 17, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ + { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/ + { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/ + { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 12, 1, 7, 6, ZSTD_fast }, /* level 0 - not used */ + { 14, 14, 14, 1, 6, 6, ZSTD_fast }, /* level 1 */ + { 14, 14, 14, 1, 4, 6, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ + { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ + { 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ + { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ + { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/ + { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/ + { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/ +}, +}; + +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. +* Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) +{ + ZSTD_compressionParameters cp; + size_t const addedSize = srcSize ? 0 : 500; + U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ + if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; + cp = ZSTD_defaultCParameters[tableID][compressionLevel]; + if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ + if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; + if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; + if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; + } + cp = ZSTD_adjustCParams(cp, srcSize, dictSize); + return cp; +} + +/*! ZSTD_getParams() : +* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). +* All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); + memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + return params; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/compress/zstd_opt.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,900 @@ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/* Note : this file is intended to be included within zstd_compress.c */ + + +#ifndef ZSTD_OPT_H_91842398743 +#define ZSTD_OPT_H_91842398743 + + +#define ZSTD_FREQ_DIV 5 +#define ZSTD_MAX_PRICE (1<<30) + +/*-************************************* +* Price functions for optimal parser +***************************************/ +FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) +{ + ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1); + ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1); + ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1); + ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1); + ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); +} + + +MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) +{ + unsigned u; + + ssPtr->cachedLiterals = NULL; + ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; + + if (ssPtr->litLengthSum == 0) { + ssPtr->litSum = (2<<Litbits); + ssPtr->litLengthSum = MaxLL+1; + ssPtr->matchLengthSum = MaxML+1; + ssPtr->offCodeSum = (MaxOff+1); + ssPtr->matchSum = (2<<Litbits); + + for (u=0; u<=MaxLit; u++) + ssPtr->litFreq[u] = 2; + for (u=0; u<=MaxLL; u++) + ssPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + ssPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + ssPtr->offCodeFreq[u] = 1; + } else { + ssPtr->matchLengthSum = 0; + ssPtr->litLengthSum = 0; + ssPtr->offCodeSum = 0; + ssPtr->matchSum = 0; + ssPtr->litSum = 0; + + for (u=0; u<=MaxLit; u++) { + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litSum += ssPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) { + ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; + } + for (u=0; u<=MaxML; u++) { + ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; + ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); + } + for (u=0; u<=MaxOff; u++) { + ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; + } + } + + ZSTD_setLog2Prices(ssPtr); +} + + +FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals) +{ + U32 price, u; + + if (litLength == 0) + return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); + + /* literals */ + if (ssPtr->cachedLiterals == literals) { + U32 const additional = litLength - ssPtr->cachedLitLength; + const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; + price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; + for (u=0; u < additional; u++) + price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1); + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } else { + price = litLength * ssPtr->log2litSum; + for (u=0; u < litLength; u++) + price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ssPtr->cachedLiterals = literals; + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } + } + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1); + } + + return price; +} + + +FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra) +{ + /* offset */ + BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); + + if (!ultra && offCode >= 20) price += (offCode-19)*2; + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1); + } + + return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; +} + + +MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + U32 u; + + /* literals */ + seqStorePtr->litSum += litLength; + for (u=0; u < litLength; u++) + seqStorePtr->litFreq[literals[u]]++; + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + seqStorePtr->litLengthFreq[llCode]++; + seqStorePtr->litLengthSum++; + } + + /* match offset */ + { BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + seqStorePtr->offCodeSum++; + seqStorePtr->offCodeFreq[offCode]++; + } + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + seqStorePtr->matchLengthFreq[mlCode]++; + seqStorePtr->matchLengthSum++; + } + + ZSTD_setLog2Prices(seqStorePtr); +} + + +#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ + opt[pos].mlen = mlen_; \ + opt[pos].off = offset_; \ + opt[pos].litlen = litlen_; \ + opt[pos].price = price_; \ + } + + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; + const BYTE* const base = zc->base; + U32 idx = zc->nextToUpdate3; + const U32 target = zc->nextToUpdate3 = (U32)(ip - base); + const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +static U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + U32 nbCompares, const U32 mls, + U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) +{ + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 hashLog = zc->params.cParams.hashLog; + const size_t h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const hashTable = zc->hashTable; + U32 matchIndex = hashTable[h]; + U32* const bt = zc->chainTable; + const U32 btLog = zc->params.cParams.chainLog - 1; + const U32 btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatchLen-1; + + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3; + matches[mnum].len = (U32)currentMl; + mnum++; + if (currentMl > ZSTD_OPT_NUM) goto update; + if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ + } + } + } + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) { + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; + } + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex; + matches[mnum].len = (U32)matchLength; + mnum++; + if (matchLength > ZSTD_OPT_NUM) break; + if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + +update: + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return mnum; +} + + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + + +/*-******************************* +* Optimal parser +*********************************/ +FORCE_INLINE +void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const BYTE* const prefixStart = base + ctx->dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + U32 offset, rep[ZSTD_REP_NUM]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(seqStorePtr); + ip += (ip==prefixStart); + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i=(ip == anchor); i<last_i; i++) { + const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i]; + if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart)) + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + best_off = i - (ip == anchor); + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); + + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + /* set prices using matches at position = 0 */ + best_mlen = (last_pos) ? last_pos : minMatch; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ + mlen++; + } } + + if (last_pos < minMatch) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } + opt[0].mlen = 1; + opt[0].litlen = litlen; + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */ + const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; + if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart)) + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); + + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos;) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength==0) offset--; + } + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; } + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 lowestIndex = ctx->lowLimit; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + + /* init */ + U32 offset, rep[ZSTD_REP_NUM]; + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } + + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(seqStorePtr); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + U32 current = (U32)(ip-base); + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + opt[0].litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i = (ip==anchor); i<last_i; i++) { + const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i]; + const U32 repIndex = (U32)(current - repCur); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( (repCur > 0 && repCur <= (S32)current) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + + best_off = i - (ip==anchor); + litlen = opt[0].litlen; + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ + + if (!last_pos && !match_num) { ip++; continue; } + + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } + opt[0].mlen = 1; + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + best_mlen = (last_pos) ? last_pos : minMatch; + + /* set prices using matches at position = 0 */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + litlen = opt[0].litlen; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + mlen++; + } } + + if (last_pos < minMatch) { + ip++; continue; + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i = (mlen != 1); i<last_i; i++) { + const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; + const U32 repIndex = (U32)(current+cur - repCur); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( (repCur > 0 && repCur <= (S32)(current+cur)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); + + if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } /* for (cur = 1; cur <= last_pos; cur++) */ + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos; ) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + + if (litLength==0) offset--; + } + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; } + + /* Last Literals */ + { size_t lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + +#endif /* ZSTD_OPT_H_91842398743 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/decompress/huf_decompress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,883 @@ +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) || defined(__GNUC__) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include <string.h> /* memcpy, memset */ +#include "bitstream.h" /* BIT_* */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Prepare ranks */ + { U32 n, nextRankStart = 0; + for (n=1; n<tableLog+1; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill DTable */ + { U32 n; + for (n=0; n<nbSymbols; n++) { + U32 const w = huffWeight[n]; + U32 const length = (1 << w) >> 1; + U32 i; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } } + + return iSize; +} + + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUF_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUF_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ + const U32 symbol = sortedSymbols[s].symbol; + const U32 weight = sortedSymbols[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 length = 1 << (sizeLog-nbBits); + const U32 start = rankVal[weight]; + U32 i = start; + const U32 end = start + length; + + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ + + rankVal[weight] += length; + } } +} + +typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s<sortedListSize; s++) { + const U16 symbol = sortedList[s].symbol; + const U32 weight = sortedList[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 start = rankVal[weight]; + const U32 length = 1 << (targetLog-nbBits); + + if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUF_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; + U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ + if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; s<nbSymbols; s++) { + U32 const w = weightList[s]; + U32 const r = rankStart[w]++; + sortedSymbol[r].symbol = (BYTE)s; + sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankVal; + nextRankVal += rankStats[w] << (w+rescale); + rankVal0[w] = current; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUF_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUF_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUF_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/decompress/zbuff_decompress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#include <stdlib.h> +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" + + +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; + +/* *** Resource management *** */ +struct ZBUFF_DCtx_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + ZBUFF_dStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTD_customMem customMem; +}; /* typedef'd to ZBUFF_DCtx within "zbuff.h" */ + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + return ZBUFF_createDCtx_advanced(defaultCustomMem); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZBUFF_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + memset(zbd, 0, sizeof(ZBUFF_DCtx)); + memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); + zbd->zd = ZSTD_createDCtx_advanced(customMem); + if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; } + zbd->stage = ZBUFFds_init; + return zbd; +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) +{ + if (zbd==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zbd->zd); + if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) +{ + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); +} + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) +{ + return ZBUFF_decompressInitDictionary(zbd, NULL, 0); +} + + +/* internal util function */ +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + U32 someMoreWork = 1; + + while (someMoreWork) { + switch(zbd->stage) + { + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (ZSTD_isError(hSize)) return hSize; + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; + } } + + zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zbd->fParams.windowSize + blockSize; + zbd->blockSize = blockSize; + if (zbd->inBuffSize < blockSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + zbd->inBuffSize = blockSize; + zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); + if (zbd->inBuff == NULL) return ERROR(memory_allocation); + } + if (zbd->outBuffSize < neededOutSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->outBuffSize = neededOutSize; + zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); + if (zbd->outBuff == NULL) return ERROR(memory_allocation); + } } + zbd->stage = ZBUFFds_read; + /* pass-through */ + + case ZBUFFds_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (neededInSize==0) { /* end of frame */ + zbd->stage = ZBUFFds_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zbd->stage = ZBUFFds_load; + /* pass-through */ + } + + case ZBUFFds_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zbd->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zbd->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + /* pass-through */ + } } + + case ZBUFFds_flush: + { size_t const toFlushSize = zbd->outEnd - zbd->outStart; + size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); + op += flushedSize; + zbd->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zbd->stage = ZBUFFds_read; + if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) + zbd->outStart = zbd->outEnd = 0; + break; + } + /* cannot flush everything */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + *srcSizePtr = ip-istart; + *dstCapacityPtr = op-ostart; + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block); + if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zbd->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/decompress/zstd_decompress.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,1842 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() will allocate memory, + * in memory stack (0), or in memory heap (1, requires malloc()) + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. +*/ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include <string.h> /* memcpy, memmove, memset */ +#include "mem.h" /* low level memory routines */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "zstd_legacy.h" +#endif + + +/*-************************************* +* Macros +***************************************/ +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* +* Context management +***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +struct ZSTD_DCtx_s +{ + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; + FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + const void* previousDstEnd; + const void* base; + const void* vBase; + const void* dictEnd; + size_t expected; + U32 rep[ZSTD_REP_NUM]; + ZSTD_frameParams fParams; + blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + U32 dictID; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litBufSize; + size_t litSize; + size_t rleSize; + BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); } + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + dctx->expected = ZSTD_frameHeaderSize_prefix; + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + dctx->vBase = NULL; + dctx->dictEnd = NULL; + dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->LLTable; + dctx->MLTptr = dctx->MLTable; + dctx->OFTptr = dctx->OFTable; + dctx->HUFptr = dctx->hufTable; + return 0; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); + if (!dctx) return NULL; + memcpy(&dctx->customMem, &customMem, sizeof(customMem)); + ZSTD_decompressBegin(dctx); + return dctx; +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + return ZSTD_createDCtx_advanced(defaultCustomMem); +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + ZSTD_free(dctx, dctx->customMem); + return 0; /* reserved as a potential error code in the future */ +} + +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; + memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ +} + +static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + ZSTD_decompressBegin(dstDCtx); /* init */ + if (srcDCtx) { /* support refDCtx on NULL */ + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; + } +} + + +/*-************************************************************* +* Decompression section +***************************************************************/ + +/* See compression format details in : doc/zstd_compression_format.md */ + +/** ZSTD_frameHeaderSize() : +* srcSize must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + + +/** ZSTD_getFrameParams() : +* decode Frame Header, or require larger `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, `srcSize` is too small, result is expected `srcSize`, +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + + if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix; + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: /* impossible */ + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } + return 0; +} + + +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not present within frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); +#endif + { ZSTD_frameParams fparams; + size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + +/** ZSTD_decodeFrameHeader() : +* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). +* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); + if (ZSTD_isError(result)) return result; /* invalid header */ + if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + return 0; +} + + +typedef struct +{ + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; + +/*! ZSTD_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + return cSize; + } +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize) +{ + if (srcSize != 1) return ERROR(srcSize_wrong); + if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, *(const BYTE*)src, regenSize); + return regenSize; +} + +/*! ZSTD_decodeLiteralsBlock() : + @return : nb of bytes read from src (< srcSize ) */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + /* fall-through */ + case set_compressed: + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + if (HUF_isError((litEncType==set_repeat) ? + ( singleStream ? + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : + ( singleStream ? + HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : + HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litBufSize = srcSize-lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + } +} + + +typedef union { + FSE_decode_t realData; + U32 alignedBy4; +} FSE_decode_t4; + +static const FSE_decode_t4 LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = { + { { LL_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ + { { 0, 0, 4 } }, /* 0 : base, symbol, bits */ + { { 16, 0, 4 } }, + { { 32, 1, 5 } }, + { { 0, 3, 5 } }, + { { 0, 4, 5 } }, + { { 0, 6, 5 } }, + { { 0, 7, 5 } }, + { { 0, 9, 5 } }, + { { 0, 10, 5 } }, + { { 0, 12, 5 } }, + { { 0, 14, 6 } }, + { { 0, 16, 5 } }, + { { 0, 18, 5 } }, + { { 0, 19, 5 } }, + { { 0, 21, 5 } }, + { { 0, 22, 5 } }, + { { 0, 24, 5 } }, + { { 32, 25, 5 } }, + { { 0, 26, 5 } }, + { { 0, 27, 6 } }, + { { 0, 29, 6 } }, + { { 0, 31, 6 } }, + { { 32, 0, 4 } }, + { { 0, 1, 4 } }, + { { 0, 2, 5 } }, + { { 32, 4, 5 } }, + { { 0, 5, 5 } }, + { { 32, 7, 5 } }, + { { 0, 8, 5 } }, + { { 32, 10, 5 } }, + { { 0, 11, 5 } }, + { { 0, 13, 6 } }, + { { 32, 16, 5 } }, + { { 0, 17, 5 } }, + { { 32, 19, 5 } }, + { { 0, 20, 5 } }, + { { 32, 22, 5 } }, + { { 0, 23, 5 } }, + { { 0, 25, 4 } }, + { { 16, 25, 4 } }, + { { 32, 26, 5 } }, + { { 0, 28, 6 } }, + { { 0, 30, 6 } }, + { { 48, 0, 4 } }, + { { 16, 1, 4 } }, + { { 32, 2, 5 } }, + { { 32, 3, 5 } }, + { { 32, 5, 5 } }, + { { 32, 6, 5 } }, + { { 32, 8, 5 } }, + { { 32, 9, 5 } }, + { { 32, 11, 5 } }, + { { 32, 12, 5 } }, + { { 0, 15, 6 } }, + { { 32, 17, 5 } }, + { { 32, 18, 5 } }, + { { 32, 20, 5 } }, + { { 32, 21, 5 } }, + { { 32, 23, 5 } }, + { { 32, 24, 5 } }, + { { 0, 35, 6 } }, + { { 0, 34, 6 } }, + { { 0, 33, 6 } }, + { { 0, 32, 6 } }, +}; /* LL_defaultDTable */ + +static const FSE_decode_t4 ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = { + { { ML_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ + { { 0, 0, 6 } }, /* 0 : base, symbol, bits */ + { { 0, 1, 4 } }, + { { 32, 2, 5 } }, + { { 0, 3, 5 } }, + { { 0, 5, 5 } }, + { { 0, 6, 5 } }, + { { 0, 8, 5 } }, + { { 0, 10, 6 } }, + { { 0, 13, 6 } }, + { { 0, 16, 6 } }, + { { 0, 19, 6 } }, + { { 0, 22, 6 } }, + { { 0, 25, 6 } }, + { { 0, 28, 6 } }, + { { 0, 31, 6 } }, + { { 0, 33, 6 } }, + { { 0, 35, 6 } }, + { { 0, 37, 6 } }, + { { 0, 39, 6 } }, + { { 0, 41, 6 } }, + { { 0, 43, 6 } }, + { { 0, 45, 6 } }, + { { 16, 1, 4 } }, + { { 0, 2, 4 } }, + { { 32, 3, 5 } }, + { { 0, 4, 5 } }, + { { 32, 6, 5 } }, + { { 0, 7, 5 } }, + { { 0, 9, 6 } }, + { { 0, 12, 6 } }, + { { 0, 15, 6 } }, + { { 0, 18, 6 } }, + { { 0, 21, 6 } }, + { { 0, 24, 6 } }, + { { 0, 27, 6 } }, + { { 0, 30, 6 } }, + { { 0, 32, 6 } }, + { { 0, 34, 6 } }, + { { 0, 36, 6 } }, + { { 0, 38, 6 } }, + { { 0, 40, 6 } }, + { { 0, 42, 6 } }, + { { 0, 44, 6 } }, + { { 32, 1, 4 } }, + { { 48, 1, 4 } }, + { { 16, 2, 4 } }, + { { 32, 4, 5 } }, + { { 32, 5, 5 } }, + { { 32, 7, 5 } }, + { { 32, 8, 5 } }, + { { 0, 11, 6 } }, + { { 0, 14, 6 } }, + { { 0, 17, 6 } }, + { { 0, 20, 6 } }, + { { 0, 23, 6 } }, + { { 0, 26, 6 } }, + { { 0, 29, 6 } }, + { { 0, 52, 6 } }, + { { 0, 51, 6 } }, + { { 0, 50, 6 } }, + { { 0, 49, 6 } }, + { { 0, 48, 6 } }, + { { 0, 47, 6 } }, + { { 0, 46, 6 } }, +}; /* ML_defaultDTable */ + +static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = { + { { OF_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ + { { 0, 0, 5 } }, /* 0 : base, symbol, bits */ + { { 0, 6, 4 } }, + { { 0, 9, 5 } }, + { { 0, 15, 5 } }, + { { 0, 21, 5 } }, + { { 0, 3, 5 } }, + { { 0, 7, 4 } }, + { { 0, 12, 5 } }, + { { 0, 18, 5 } }, + { { 0, 23, 5 } }, + { { 0, 5, 5 } }, + { { 0, 8, 4 } }, + { { 0, 14, 5 } }, + { { 0, 20, 5 } }, + { { 0, 2, 5 } }, + { { 16, 7, 4 } }, + { { 0, 11, 5 } }, + { { 0, 17, 5 } }, + { { 0, 22, 5 } }, + { { 0, 4, 5 } }, + { { 16, 8, 4 } }, + { { 0, 13, 5 } }, + { { 0, 19, 5 } }, + { { 0, 1, 5 } }, + { { 16, 6, 4 } }, + { { 0, 10, 5 } }, + { { 0, 16, 5 } }, + { { 0, 28, 5 } }, + { { 0, 27, 5 } }, + { { 0, 26, 5 } }, + { { 0, 25, 5 } }, + { { 0, 24, 5 } }, +}; /* OF_defaultDTable */ + +/*! ZSTD_buildSeqTable() : + @return : nb bytes read from src, + or an error code if it fails, testable with ZSTD_isError() +*/ +static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr, + symbolEncodingType_e type, U32 max, U32 maxLog, + const void* src, size_t srcSize, + const FSE_decode_t4* defaultTable, U32 flagRepeatTable) +{ + const void* const tmpPtr = defaultTable; /* bypass strict aliasing */ + switch(type) + { + case set_rle : + if (!srcSize) return ERROR(srcSize_wrong); + if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); + FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src); + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = (const FSE_DTable*)tmpPtr; + return 0; + case set_repeat: + if (!flagRepeatTable) return ERROR(corruption_detected); + return 0; + default : /* impossible */ + case set_compressed : + { U32 tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSE_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + FSE_buildDTable(DTableSpace, norm, max, tableLog); + *DTablePtr = DTableSpace; + return headerSize; + } } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + { int nbSeq = *ip++; + if (!nbSeq) { *nbSeqPtr=0; return 1; } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + } else { + if (ip >= iend) return ERROR(srcSize_wrong); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + } + + /* FSE table descriptors */ + if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + BIT_DStream_t DStream; + FSE_DState_t stateLL; + FSE_DState_t stateOffb; + FSE_DState_t stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + + +static seq_t ZSTD_decodeSequence(seqState_t* seqState) +{ + seq_t seq; + + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BIT_readBits(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits() || + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); + + /* ANS state update */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + + +FORCE_NOINLINE +size_t ZSTD_execSequenceLast7(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ + + /* copy literals */ + if (op < oend_w) { + ZSTD_wildcopy(op, *litPtr, oend_w - op); + *litPtr += oend_w - op; + op = oend_w; + } + while (op < oLitEnd) *op++ = *(*litPtr)++; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + while (op < oMatchEnd) *op++ = *match++; + return sequenceLength; +} + + +FORCE_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + if (op > oend_w) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + /* Requirement: op <= oend_w */ + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +static size_t ZSTD_decompressSequences( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTD_decodeSequence(&seqState); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + + +static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + + if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); + + /* Decode literals sub-block */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); +} + + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + +/*! ZSTD_decompressFrame() : +* `dctx` must be properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize)); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + return ERROR(corruption_detected); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + if (remainingSize<4) return ERROR(checksum_wrong); + checkRead = MEM_readLE32(ip); + if (checkRead != checkCalc) return ERROR(checksum_wrong); + remainingSize -= 4; + } + + if (remainingSize) return ERROR(srcSize_wrong); + return op-ostart; +} + + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); +#endif + ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (dctx==NULL) return ERROR(memory_allocation); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ + +/** ZSTD_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + /* Sanity check */ + if (srcSize != dctx->expected) return ERROR(srcSize_wrong); + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + } + dctx->expected = 0; /* not necessary to copy more */ + + case ZSTDds_decodeFrameHeader: + memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); + CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = 3; /* go directly to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + break; + case bt_raw : + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); + break; + case bt_reserved : /* should never happen */ + default: + return ERROR(corruption_detected); + } + if (ZSTD_isError(rSize)) return rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + } + return rSize; + } + case ZSTDds_checkChecksum: + { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ + if (check32 != h32) return ERROR(checksum_wrong); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + default: + return ERROR(GENERIC); /* impossible */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + { size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize); + if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue=MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + dctx->litEntropy = dctx->fseEntropy = 1; + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_DICT_MAGIC) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + 4); + + /* load entropy tables */ + dict = (const char*)dict + 8; + dictSize -= 8; + { size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize); + if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + CHECK_F(ZSTD_decompressBegin(dctx)); + if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +struct ZSTD_DDict_s { + void* dict; + size_t dictSize; + ZSTD_DCtx* refContext; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); + return NULL; + } + + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } + { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTD_isError(errorCode)) { + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); + return NULL; + } } + + ddict->dict = dictContent; + ddict->dictSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression without startup delay. +* `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->refContext->customMem; + ZSTD_freeDCtx(ddict->refContext); + ZSTD_free(ddict->dict, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); +#endif + ZSTD_refDCtx(dctx, ddict->refContext); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +typedef enum { zdss_init, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +/* *** Resource management *** */ +struct ZSTD_DStream_s { + ZSTD_DCtx* dctx; + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; + ZSTD_frameParams fParams; + ZSTD_dStreamStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ + size_t lhSize; + ZSTD_customMem customMem; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; +}; /* typedef'd to ZSTD_DStream within "zstd.h" */ + + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(defaultCustomMem); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_DStream* zds; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem); + if (zds==NULL) return NULL; + memset(zds, 0, sizeof(ZSTD_DStream)); + memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); + zds->dctx = ZSTD_createDCtx_advanced(customMem); + if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->stage = zdss_init; + zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + return zds; +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support free on null */ + { ZSTD_customMem const cMem = zds->customMem; + ZSTD_freeDCtx(zds->dctx); + ZSTD_freeDDict(zds->ddictLocal); + ZSTD_free(zds->inBuff, cMem); + ZSTD_free(zds->outBuff, cMem); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (zds->legacyContext) + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); +#endif + ZSTD_free(zds, cMem); + return 0; + } +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + ZSTD_freeDDict(zds->ddictLocal); + if (dict) { + zds->ddictLocal = ZSTD_createDDict(dict, dictSize); + if (zds->ddictLocal == NULL) return ERROR(memory_allocation); + } else zds->ddictLocal = NULL; + zds->ddict = zds->ddictLocal; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */ +{ + size_t const initResult = ZSTD_initDStream(zds); + zds->ddict = ddict; + return initResult; +} + +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + switch(paramType) + { + default : return ERROR(parameter_unknown); + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize; +} + + +/* ***** Decompression ***** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; + char* op = ostart; + U32 someMoreWork = 1; + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); +#endif + + while (someMoreWork) { + switch(zds->stage) + { + case zdss_init : + return ERROR(init_missing); + + case zdss_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + const void* const dict = zds->ddict ? zds->ddict->dict : NULL; + size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; + CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, + dict, dictSize)); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + } else { + return hSize; /* error */ + } } +#else + return hSize; +#endif + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + input->pos = input->size; + return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL; + ZSTD_refDCtx(zds->dctx, refContext); + } + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = blockSize; + zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + if (zds->outBuffSize < neededOutSize) { + ZSTD_free(zds->outBuff, zds->customMem); + zds->outBuffSize = neededOutSize; + zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + } } + zds->stage = zdss_read; + /* pass-through */ + + case zdss_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); + if (neededInSize==0) { /* end of frame */ + zds->stage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->stage = zdss_load; + /* pass-through */ + } + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + /* pass-through */ + } } + + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->stage = zdss_read; + if (zds->outStart + zds->blockSize > zds->outBuffSize) + zds->outStart = zds->outEnd = 0; + break; + } + /* cannot complete flush */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->pos += (size_t)(ip-istart); + output->pos += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ + input->pos++; /* release hostage */ + } + return 0; + } + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/dictBuilder/divsufsort.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,1913 @@ +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/*- Compiler specifics -*/ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wshorten-64-to-32" +#endif + +#if defined(_MSC_VER) +# pragma warning(disable : 4244) +# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */ +#endif + + +/*- Dependencies -*/ +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> + +#include "divsufsort.h" + +/*- Constants -*/ +#if defined(INLINE) +# undef INLINE +#endif +#if !defined(INLINE) +# define INLINE __inline +#endif +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (256) +#endif +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# define SS_MISORT_STACKSIZE (96) +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#define SS_SMERGE_STACKSIZE (32) +#define TR_INSERTIONSORT_THRESHOLD (8) +#define TR_STACKSIZE (64) + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Functions -*/ + +static const int lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +int +ss_ilg(int n) { +#if SS_BLOCKSIZE == 0 + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +#elif SS_BLOCKSIZE < 256 + return lg_table[n]; +#else + return (n & 0xff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]; +#endif +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + +#if SS_BLOCKSIZE != 0 + +static const int sqq_table[256] = { + 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, + 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, + 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, +110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, +128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, +156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, +169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, +181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, +192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, +202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, +212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, +221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, +230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, +239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, +247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 +}; + +static INLINE +int +ss_isqrt(int x) { + int y, e; + + if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; } + e = (x & 0xffff0000) ? + ((x & 0xff000000) ? + 24 + lg_table[(x >> 24) & 0xff] : + 16 + lg_table[(x >> 16) & 0xff]) : + ((x & 0x0000ff00) ? + 8 + lg_table[(x >> 8) & 0xff] : + 0 + lg_table[(x >> 0) & 0xff]); + + if(e >= 16) { + y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); + if(e >= 24) { y = (y + 1 + x / y) >> 1; } + y = (y + 1 + x / y) >> 1; + } else if(e >= 8) { + y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; + } else { + return sqq_table[x] >> 4; + } + + return (x < (y * y)) ? y - 1 : y; +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Compares two suffixes. */ +static INLINE +int +ss_compare(const unsigned char *T, + const int *p1, const int *p2, + int depth) { + const unsigned char *U1, *U2, *U1n, *U2n; + + for(U1 = T + depth + *p1, + U2 = T + depth + *p2, + U1n = T + *(p1 + 1) + 2, + U2n = T + *(p2 + 1) + 2; + (U1 < U1n) && (U2 < U2n) && (*U1 == *U2); + ++U1, ++U2) { + } + + return U1 < U1n ? + (U2 < U2n ? *U1 - *U2 : 1) : + (U2 < U2n ? -1 : 0); +} + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) + +/* Insertionsort for small size groups */ +static +void +ss_insertionsort(const unsigned char *T, const int *PA, + int *first, int *last, int depth) { + int *i, *j; + int t; + int r; + + for(i = last - 2; first <= i; --i) { + for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) { + do { *(j - 1) = *j; } while((++j < last) && (*j < 0)); + if(last <= j) { break; } + } + if(r == 0) { *j = ~*j; } + *(j - 1) = t; + } +} + +#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */ + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +void +ss_fixdown(const unsigned char *Td, const int *PA, + int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = Td[PA[SA[k = j++]]]; + if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + ss_fixdown(Td, PA, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +ss_median3(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3) { + int *t; + if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); } + if(Td[PA[*v2]] > Td[PA[*v3]]) { + if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +ss_median5(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); } + if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); } + if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); } + if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); } + if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); } + if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return ss_median3(Td, PA, first, middle, last - 1); + } else { + t >>= 2; + return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = ss_median3(Td, PA, first, first + t, first + (t << 1)); + middle = ss_median3(Td, PA, middle - t, middle, middle + t); + last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1); + return ss_median3(Td, PA, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +/* Binary partition for substrings. */ +static INLINE +int * +ss_partition(const int *PA, + int *first, int *last, int depth) { + int *a, *b; + int t; + for(a = first - 1, b = last;;) { + for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; } + for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { } + if(b <= a) { break; } + t = ~*b; + *b = *a; + *a = t; + } + if(first < a) { *first = ~*first; } + return a; +} + +/* Multikey introsort for medium size groups. */ +static +void +ss_mintrosort(const unsigned char *T, const int *PA, + int *first, int *last, + int depth) { +#define STACK_SIZE SS_MISORT_STACKSIZE + struct { int *a, *b, c; int d; } stack[STACK_SIZE]; + const unsigned char *Td; + int *a, *b, *c, *d, *e, *f; + int s, t; + int ssize; + int limit; + int v, x = 0; + + for(ssize = 0, limit = ss_ilg(last - first);;) { + + if((last - first) <= SS_INSERTIONSORT_THRESHOLD) { +#if 1 < SS_INSERTIONSORT_THRESHOLD + if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); } +#endif + STACK_POP(first, last, depth, limit); + continue; + } + + Td = T + depth; + if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); } + if(limit < 0) { + for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) { + if((x = Td[PA[*a]]) != v) { + if(1 < (a - first)) { break; } + v = x; + first = a; + } + } + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, a, depth); + } + if((a - first) <= (last - a)) { + if(1 < (a - first)) { + STACK_PUSH(a, last, depth, -1); + last = a, depth += 1, limit = ss_ilg(a - first); + } else { + first = a, limit = -1; + } + } else { + if(1 < (last - a)) { + STACK_PUSH(first, a, depth + 1, ss_ilg(a - first)); + first = a, limit = -1; + } else { + last = a, depth += 1, limit = ss_ilg(a - first); + } + } + continue; + } + + /* choose pivot */ + a = ss_pivot(Td, PA, first, last); + v = Td[PA[*a]]; + SWAP(*first, *a); + + /* partition */ + for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + + a = first + (b - a), c = last - (d - c); + b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth); + + if((a - first) <= (last - c)) { + if((last - c) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(c, last, depth, limit); + last = a; + } else if((a - first) <= (c - b)) { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + last = a; + } else { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(first, a, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } else { + if((a - first) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(first, a, depth, limit); + first = c; + } else if((last - c) <= (c - b)) { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + first = c; + } else { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(c, last, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } + } else { + limit += 1; + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, last, depth); + limit = ss_ilg(last - first); + } + depth += 1; + } + } +#undef STACK_SIZE +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + + +/*---------------------------------------------------------------------------*/ + +#if SS_BLOCKSIZE != 0 + +static INLINE +void +ss_blockswap(int *a, int *b, int n) { + int t; + for(; 0 < n; --n, ++a, ++b) { + t = *a, *a = *b, *b = t; + } +} + +static INLINE +void +ss_rotate(int *first, int *middle, int *last) { + int *a, *b, t; + int l, r; + l = middle - first, r = last - middle; + for(; (0 < l) && (0 < r);) { + if(l == r) { ss_blockswap(first, middle, l); break; } + if(l < r) { + a = last - 1, b = middle - 1; + t = *a; + do { + *a-- = *b, *b-- = *a; + if(b < first) { + *a = t; + last = a; + if((r -= l + 1) <= l) { break; } + a -= 1, b = middle - 1; + t = *a; + } + } while(1); + } else { + a = first, b = middle; + t = *a; + do { + *a++ = *b, *b++ = *a; + if(last <= b) { + *a = t; + first = a + 1; + if((l -= r + 1) <= r) { break; } + a += 1, b = middle; + t = *a; + } + } while(1); + } + } +} + + +/*---------------------------------------------------------------------------*/ + +static +void +ss_inplacemerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int depth) { + const int *p; + int *a, *b; + int len, half; + int q, r; + int x; + + for(;;) { + if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); } + else { x = 0; p = PA + *(last - 1); } + for(a = first, len = middle - first, half = len >> 1, r = -1; + 0 < len; + len = half, half >>= 1) { + b = a + half; + q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth); + if(q < 0) { + a = b + 1; + half -= (len & 1) ^ 1; + } else { + r = q; + } + } + if(a < middle) { + if(r == 0) { *a = ~*a; } + ss_rotate(a, middle, last); + last -= middle - a; + middle = a; + if(first == middle) { break; } + } + --last; + if(x != 0) { while(*--last < 0) { } } + if(middle == last) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Merge-forward with internal buffer. */ +static +void +ss_mergeforward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + int *a, *b, *c, *bufend; + int t; + int r; + + bufend = buf + (middle - first) - 1; + ss_blockswap(buf, first, middle - first); + + for(t = *(a = first), b = buf, c = middle;;) { + r = ss_compare(T, PA + *b, PA + *c, depth); + if(r < 0) { + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + } else if(r > 0) { + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } else { + *c = ~*c; + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } + } +} + +/* Merge-backward with internal buffer. */ +static +void +ss_mergebackward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + const int *p1, *p2; + int *a, *b, *c, *bufend; + int t; + int r; + int x; + + bufend = buf + (last - middle) - 1; + ss_blockswap(buf, middle, last - middle); + + x = 0; + if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; } + else { p1 = PA + *bufend; } + if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; } + else { p2 = PA + *(middle - 1); } + for(t = *(a = last - 1), b = bufend, c = middle - 1;;) { + r = ss_compare(T, p1, p2, depth); + if(0 < r) { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = *b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + } else if(r < 0) { + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } else { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = ~*b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } + } +} + +/* D&C based merge. */ +static +void +ss_swapmerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int bufsize, int depth) { +#define STACK_SIZE SS_SMERGE_STACKSIZE +#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a))) +#define MERGE_CHECK(a, b, c)\ + do {\ + if(((c) & 1) ||\ + (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\ + *(a) = ~*(a);\ + }\ + if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\ + *(b) = ~*(b);\ + }\ + } while(0) + struct { int *a, *b, *c; int d; } stack[STACK_SIZE]; + int *l, *r, *lm, *rm; + int m, len, half; + int ssize; + int check, next; + + for(check = 0, ssize = 0;;) { + if((last - middle) <= bufsize) { + if((first < middle) && (middle < last)) { + ss_mergebackward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + if((middle - first) <= bufsize) { + if(first < middle) { + ss_mergeforward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1) { + if(ss_compare(T, PA + GETIDX(*(middle + m + half)), + PA + GETIDX(*(middle - m - half - 1)), depth) < 0) { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if(0 < m) { + lm = middle - m, rm = middle + m; + ss_blockswap(lm, middle, m); + l = r = middle, next = 0; + if(rm < last) { + if(*rm < 0) { + *rm = ~*rm; + if(first < lm) { for(; *--l < 0;) { } next |= 4; } + next |= 1; + } else if(first < lm) { + for(; *r < 0; ++r) { } + next |= 2; + } + } + + if((l - first) <= (last - r)) { + STACK_PUSH(r, rm, last, (next & 3) | (check & 4)); + middle = lm, last = l, check = (check & 3) | (next & 4); + } else { + if((next & 2) && (r == middle)) { next ^= 6; } + STACK_PUSH(first, lm, l, (check & 3) | (next & 4)); + first = r, middle = rm, check = (next & 3) | (check & 4); + } + } else { + if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) { + *middle = ~*middle; + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + } + } +#undef STACK_SIZE +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Substring sort */ +static +void +sssort(const unsigned char *T, const int *PA, + int *first, int *last, + int *buf, int bufsize, + int depth, int n, int lastsuffix) { + int *a; +#if SS_BLOCKSIZE != 0 + int *b, *middle, *curbuf; + int j, k, curbufsize, limit; +#endif + int i; + + if(lastsuffix != 0) { ++first; } + +#if SS_BLOCKSIZE == 0 + ss_mintrosort(T, PA, first, last, depth); +#else + if((bufsize < SS_BLOCKSIZE) && + (bufsize < (last - first)) && + (bufsize < (limit = ss_isqrt(last - first)))) { + if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; } + buf = middle = last - limit, bufsize = limit; + } else { + middle = last, limit = 0; + } + for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth); +#endif + curbufsize = last - (a + SS_BLOCKSIZE); + curbuf = a + SS_BLOCKSIZE; + if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; } + for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) { + ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth); + } + } +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, middle, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, middle, depth); +#endif + for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { + if(i & 1) { + ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth); + a -= k; + } + } + if(limit != 0) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, middle, last, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, middle, last, depth); +#endif + ss_inplacemerge(T, PA, first, middle, last, depth); + } +#endif + + if(lastsuffix != 0) { + /* Insert last type B* suffix. */ + int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2; + for(a = first, i = *(first - 1); + (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth))); + ++a) { + *(a - 1) = *a; + } + *(a - 1) = i; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +int +tr_ilg(int n) { + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +} + + +/*---------------------------------------------------------------------------*/ + +/* Simple insertionsort for small size groups. */ +static +void +tr_insertionsort(const int *ISAd, int *first, int *last) { + int *a, *b; + int t, r; + + for(a = first + 1; a < last; ++a) { + for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) { + do { *(b + 1) = *b; } while((first <= --b) && (*b < 0)); + if(b < first) { break; } + } + if(r == 0) { *b = ~*b; } + *(b + 1) = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_fixdown(const int *ISAd, int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = ISAd[SA[k = j++]]; + if(d < (e = ISAd[SA[j]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +tr_heapsort(const int *ISAd, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + tr_fixdown(ISAd, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +tr_median3(const int *ISAd, int *v1, int *v2, int *v3) { + int *t; + if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); } + if(ISAd[*v2] > ISAd[*v3]) { + if(ISAd[*v1] > ISAd[*v3]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +tr_median5(const int *ISAd, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); } + if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); } + if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); } + if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); } + if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); } + if(ISAd[*v3] > ISAd[*v4]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +tr_pivot(const int *ISAd, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return tr_median3(ISAd, first, middle, last - 1); + } else { + t >>= 2; + return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = tr_median3(ISAd, first, first + t, first + (t << 1)); + middle = tr_median3(ISAd, middle - t, middle, middle + t); + last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1); + return tr_median3(ISAd, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +typedef struct _trbudget_t trbudget_t; +struct _trbudget_t { + int chance; + int remain; + int incval; + int count; +}; + +static INLINE +void +trbudget_init(trbudget_t *budget, int chance, int incval) { + budget->chance = chance; + budget->remain = budget->incval = incval; +} + +static INLINE +int +trbudget_check(trbudget_t *budget, int size) { + if(size <= budget->remain) { budget->remain -= size; return 1; } + if(budget->chance == 0) { budget->count += size; return 0; } + budget->remain += budget->incval - size; + budget->chance -= 1; + return 1; +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_partition(const int *ISAd, + int *first, int *middle, int *last, + int **pa, int **pb, int v) { + int *a, *b, *c, *d, *e, *f; + int t, s; + int x = 0; + + for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + first += (b - a), last -= (d - c); + } + *pa = first, *pb = last; +} + +static +void +tr_copy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + /* sort suffixes of middle partition + by using sorted order of suffixes of left and right partition. */ + int *c, *d, *e; + int s, v; + + v = b - SA - 1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + ISA[s] = d - SA; + } + } + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + ISA[s] = d - SA; + } + } +} + +static +void +tr_partialcopy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + int *c, *d, *e; + int s, v; + int rank, lastrank, newrank = -1; + + v = b - SA - 1; + lastrank = -1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } + + lastrank = -1; + for(e = d; first <= e; --e) { + rank = ISA[*e]; + if(lastrank != rank) { lastrank = rank; newrank = e - SA; } + if(newrank != rank) { ISA[*e] = newrank; } + } + + lastrank = -1; + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } +} + +static +void +tr_introsort(int *ISA, const int *ISAd, + int *SA, int *first, int *last, + trbudget_t *budget) { +#define STACK_SIZE TR_STACKSIZE + struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE]; + int *a, *b, *c; + int t; + int v, x = 0; + int incr = ISAd - ISA; + int limit, next; + int ssize, trlink = -1; + + for(ssize = 0, limit = tr_ilg(last - first);;) { + + if(limit < 0) { + if(limit == -1) { + /* tandem repeat partition */ + tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1); + + /* update ranks */ + if(a < last) { + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + } + if(b < last) { + for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } + } + + /* push */ + if(1 < (b - a)) { + STACK_PUSH5(NULL, a, b, 0, 0); + STACK_PUSH5(ISAd - incr, first, last, -2, trlink); + trlink = ssize - 2; + } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink); + last = a, limit = tr_ilg(a - first); + } else if(1 < (last - b)) { + first = b, limit = tr_ilg(last - b); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink); + first = b, limit = tr_ilg(last - b); + } else if(1 < (a - first)) { + last = a, limit = tr_ilg(a - first); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else if(limit == -2) { + /* tandem repeat copy */ + a = stack[--ssize].b, b = stack[ssize].c; + if(stack[ssize].d == 0) { + tr_copy(ISA, SA, first, a, b, last, ISAd - ISA); + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA); + } + STACK_POP5(ISAd, first, last, limit, trlink); + } else { + /* sorted partition */ + if(0 <= *first) { + a = first; + do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a)); + first = a; + } + if(first < last) { + a = first; do { *a = ~*a; } while(*++a < 0); + next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1; + if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } } + + /* push */ + if(trbudget_check(budget, a - first)) { + if((a - first) <= (last - a)) { + STACK_PUSH5(ISAd, a, last, -3, trlink); + ISAd += incr, last = a, limit = next; + } else { + if(1 < (last - a)) { + STACK_PUSH5(ISAd + incr, first, a, next, trlink); + first = a, limit = -3; + } else { + ISAd += incr, last = a, limit = next; + } + } + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + if(1 < (last - a)) { + first = a, limit = -3; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + continue; + } + + if((last - first) <= TR_INSERTIONSORT_THRESHOLD) { + tr_insertionsort(ISAd, first, last); + limit = -3; + continue; + } + + if(limit-- == 0) { + tr_heapsort(ISAd, first, last - first); + for(a = last - 1; first < a; a = b) { + for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; } + } + limit = -3; + continue; + } + + /* choose pivot */ + a = tr_pivot(ISAd, first, last); + SWAP(*first, *a); + v = ISAd[*first]; + + /* partition */ + tr_partition(ISAd, first, first + 1, last, &a, &b, v); + if((last - first) != (b - a)) { + next = (ISA[*a] != v) ? tr_ilg(b - a) : -1; + + /* update ranks */ + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } } + + /* push */ + if((1 < (b - a)) && (trbudget_check(budget, b - a))) { + if((a - first) <= (last - b)) { + if((last - b) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((a - first) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + if((a - first) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((last - b) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } + } else { + if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + first = b; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + last = a; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } + } else { + if(trbudget_check(budget, last - first)) { + limit = tr_ilg(last - first), ISAd += incr; + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } +#undef STACK_SIZE +} + + + +/*---------------------------------------------------------------------------*/ + +/* Tandem repeat sort */ +static +void +trsort(int *ISA, int *SA, int n, int depth) { + int *ISAd; + int *first, *last; + trbudget_t budget; + int t, skip, unsorted; + + trbudget_init(&budget, tr_ilg(n) * 2 / 3, n); +/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */ + for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) { + first = SA; + skip = 0; + unsorted = 0; + do { + if((t = *first) < 0) { first -= t; skip += t; } + else { + if(skip != 0) { *(first + skip) = skip; skip = 0; } + last = SA + ISA[t] + 1; + if(1 < (last - first)) { + budget.count = 0; + tr_introsort(ISA, ISAd, SA, first, last, &budget); + if(budget.count != 0) { unsorted += budget.count; } + else { skip = first - last; } + } else if((last - first) == 1) { + skip = -1; + } + first = last; + } + } while(first < (SA + n)); + if(skip != 0) { *(first + skip) = skip; } + if(unsorted == 0) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Sorts suffixes of type B*. */ +static +int +sort_typeBstar(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int openMP) { + int *PAb, *ISAb, *buf; +#ifdef LIBBSC_OPENMP + int *curbuf; + int l; +#endif + int i, j, k, t, m, bufsize; + int c0, c1; +#ifdef LIBBSC_OPENMP + int d0, d1; +#endif + (void)openMP; + + /* Initialize bucket arrays. */ + for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; } + for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; } + + /* Count the number of occurrences of the first one or two characters of each + type A, B and B* suffix. Moreover, store the beginning position of all + type B* suffixes into the array SA. */ + for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) { + /* type A suffix. */ + do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1)); + if(0 <= i) { + /* type B* suffix. */ + ++BUCKET_BSTAR(c0, c1); + SA[--m] = i; + /* type B suffix. */ + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { + ++BUCKET_B(c0, c1); + } + } + } + m = n - m; +/* +note: + A type B* suffix is lexicographically smaller than a type B suffix that + begins with the same first two characters. +*/ + + /* Calculate the index of start/end point of each bucket. */ + for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) { + t = i + BUCKET_A(c0); + BUCKET_A(c0) = i + j; /* start point */ + i = t + BUCKET_B(c0, c0); + for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) { + j += BUCKET_BSTAR(c0, c1); + BUCKET_BSTAR(c0, c1) = j; /* end point */ + i += BUCKET_B(c0, c1); + } + } + + if(0 < m) { + /* Sort the type B* suffixes by their first two characters. */ + PAb = SA + n - m; ISAb = SA + m; + for(i = m - 2; 0 <= i; --i) { + t = PAb[i], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = i; + } + t = PAb[m - 1], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = m - 1; + + /* Sort the type B* substrings using sssort. */ +#ifdef LIBBSC_OPENMP + if (openMP) + { + buf = SA + m; + c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m; +#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1) + { + bufsize = (n - (2 * m)) / omp_get_num_threads(); + curbuf = buf + omp_get_thread_num() * bufsize; + k = 0; + for(;;) { + #pragma omp critical(sssort_lock) + { + if(0 < (l = j)) { + d0 = c0, d1 = c1; + do { + k = BUCKET_BSTAR(d0, d1); + if(--d1 <= d0) { + d1 = ALPHABET_SIZE - 1; + if(--d0 < 0) { break; } + } + } while(((l - k) <= 1) && (0 < (l = k))); + c0 = d0, c1 = d1, j = k; + } + } + if(l == 0) { break; } + sssort(T, PAb, SA + k, SA + l, + curbuf, bufsize, 2, n, *(SA + k) == (m - 1)); + } + } + } + else + { + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } + } +#else + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } +#endif + + /* Compute ranks of type B* substrings. */ + for(i = m - 1; 0 <= i; --i) { + if(0 <= SA[i]) { + j = i; + do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i])); + SA[i + 1] = i - j; + if(i <= 0) { break; } + } + j = i; + do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0); + ISAb[SA[i]] = j; + } + + /* Construct the inverse suffix array of type B* suffixes using trsort. */ + trsort(ISAb, SA, m, 1); + + /* Set the sorted order of tyoe B* suffixes. */ + for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) { + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { } + if(0 <= i) { + t = i; + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { } + SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t; + } + } + + /* Calculate the index of start/end point of each bucket. */ + BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */ + for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) { + i = BUCKET_A(c0 + 1) - 1; + for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) { + t = i - BUCKET_B(c0, c1); + BUCKET_B(c0, c1) = i; /* end point */ + + /* Move all type B* suffixes to the correct position. */ + for(i = t, j = BUCKET_BSTAR(c0, c1); + j <= k; + --i, --k) { SA[i] = SA[k]; } + } + BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */ + BUCKET_B(c0, c0) = i; /* end point */ + } + } + + return m; +} + +/* Constructs the suffix array by using the sorted order of type B* suffixes. */ +static +void +construct_SA(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + *j = ~s; + c0 = T[--s]; + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else { + assert(((s == 0) && (T[s] == c1)) || (s < 0)); + *j = ~s; + } + } + } + } + + /* Construct the suffix array by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + if((s == 0) || (T[s - 1] < c0)) { s = ~s; } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else { + assert(s < 0); + *i = ~s; + } + } +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + *i = c0; + if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT_indexes(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m, + unsigned char * num_indexes, int * indexes) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + int mod = n / 8; + { + mod |= mod >> 1; mod |= mod >> 2; + mod |= mod >> 4; mod |= mod >> 8; + mod |= mod >> 16; mod >>= 1; + + *num_indexes = (unsigned char)((n - 1) / (mod + 1)); + } + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA; + + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + if (T[n - 2] < c2) { + if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[n - 2]); + } + else { + *k++ = n - 1; + } + + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA; + + c0 = T[--s]; + *i = c0; + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + if((0 < s) && (T[s - 1] < c0)) { + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[s - 1]); + } else + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP) { + int *bucket_A, *bucket_B; + int m; + int err = 0; + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; } + else if(n == 0) { return 0; } + else if(n == 1) { SA[0] = 0; return 0; } + else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } + + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Suffixsort. */ + if((bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP); + construct_SA(T, SA, bucket_A, bucket_B, n, m); + } else { + err = -2; + } + + free(bucket_B); + free(bucket_A); + + return err; +} + +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) { + int *B; + int *bucket_A, *bucket_B; + int m, pidx, i; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0)) { return -1; } + else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + + if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); } + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Burrows-Wheeler Transform. */ + if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP); + + if (num_indexes == NULL || indexes == NULL) { + pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m); + } else { + pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes); + } + + /* Copy to output string. */ + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; } + for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; } + pidx += 1; + } else { + pidx = -2; + } + + free(bucket_B); + free(bucket_A); + if(A == NULL) { free(B); } + + return pidx; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/dictBuilder/divsufsort.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,67 @@ +/* + * divsufsort.h for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_H +#define _DIVSUFSORT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T [0..n-1] The input string. + * @param SA [0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @param openMP enables OpenMP optimization. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T [0..n-1] The input string. + * @param U [0..n-1] The output string. (can be T) + * @param A [0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param num_indexes The length of secondary indexes array. (can be NULL) + * @param indexes The secondary indexes array. (can be NULL) + * @param openMP enables OpenMP optimization. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_H */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/dictBuilder/zdict.c Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,1010 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/*-************************************** +* Tuning parameters +****************************************/ +#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) +#define ZDICT_MIN_SAMPLES_SIZE 512 + + +/*-************************************** +* Compiler Options +****************************************/ +/* Unix Large Files support (>4GB) */ +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _LARGEFILE64_SOURCE +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* memset */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <time.h> /* clock */ + +#include "mem.h" /* read */ +#include "error_private.h" +#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" /* includes zstd.h */ +#include "xxhash.h" +#include "divsufsort.h" +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define DICTLISTSIZE_DEFAULT 10000 + +#define NOISELENGTH 32 + +#define MINRATIO 4 +static const int g_compressionLevel_default = 5; +static const U32 g_selectivity_default = 9; +static const size_t g_provision_entropySize = 200; +static const size_t g_min_fast_dictContent = 192; + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ + +static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } + +static void ZDICT_printHex(const void* ptr, size_t length) +{ + const BYTE* const b = (const BYTE*)ptr; + size_t u; + for (u=0; u<length; u++) { + BYTE c = b[u]; + if (c<32 || c>126) c = '.'; /* non-printable char */ + DISPLAY("%c", c); + } +} + + +/*-******************************************************** +* Helper functions +**********************************************************/ +unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static unsigned ZDICT_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +/*! ZDICT_count() : + Count the nb of common bytes between 2 pointers. + Note : this function presumes end of buffer followed by noisy guard band. +*/ +static size_t ZDICT_count(const void* pIn, const void* pMatch) +{ + const char* const pStart = (const char*)pIn; + for (;;) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn = (const char*)pIn+sizeof(size_t); + pMatch = (const char*)pMatch+sizeof(size_t); + continue; + } + pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + return (size_t)((const char*)pIn - pStart); + } +} + + +typedef struct { + U32 pos; + U32 length; + U32 savings; +} dictItem; + +static void ZDICT_initDictItem(dictItem* d) +{ + d->pos = 1; + d->length = 0; + d->savings = (U32)(-1); +} + + +#define LLIMIT 64 /* heuristic determined experimentally */ +#define MINMATCHLENGTH 7 /* heuristic determined experimentally */ +static dictItem ZDICT_analyzePos( + BYTE* doneMarks, + const int* suffix, U32 start, + const void* buffer, U32 minRatio, U32 notificationLevel) +{ + U32 lengthList[LLIMIT] = {0}; + U32 cumulLength[LLIMIT] = {0}; + U32 savings[LLIMIT] = {0}; + const BYTE* b = (const BYTE*)buffer; + size_t length; + size_t maxLength = LLIMIT; + size_t pos = suffix[start]; + U32 end = start; + dictItem solution; + + /* init */ + memset(&solution, 0, sizeof(solution)); + doneMarks[pos] = 1; + + /* trivial repetition cases */ + if ( (MEM_read16(b+pos+0) == MEM_read16(b+pos+2)) + ||(MEM_read16(b+pos+1) == MEM_read16(b+pos+3)) + ||(MEM_read16(b+pos+2) == MEM_read16(b+pos+4)) ) { + /* skip and mark segment */ + U16 u16 = MEM_read16(b+pos+4); + U32 u, e = 6; + while (MEM_read16(b+pos+e) == u16) e+=2 ; + if (b[pos+e] == b[pos+e-1]) e++; + for (u=1; u<e; u++) + doneMarks[pos+u] = 1; + return solution; + } + + /* look forward */ + do { + end++; + length = ZDICT_count(b + pos, b + suffix[end]); + } while (length >=MINMATCHLENGTH); + + /* look backward */ + do { + length = ZDICT_count(b + pos, b + *(suffix+start-1)); + if (length >=MINMATCHLENGTH) start--; + } while(length >= MINMATCHLENGTH); + + /* exit if not found a minimum nb of repetitions */ + if (end-start < minRatio) { + U32 idx; + for(idx=start; idx<end; idx++) + doneMarks[suffix[idx]] = 1; + return solution; + } + + { int i; + U32 searchLength; + U32 refinedStart = start; + U32 refinedEnd = end; + + DISPLAYLEVEL(4, "\n"); + DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos); + DISPLAYLEVEL(4, "\n"); + + for (searchLength = MINMATCHLENGTH ; ; searchLength++) { + BYTE currentChar = 0; + U32 currentCount = 0; + U32 currentID = refinedStart; + U32 id; + U32 selectedCount = 0; + U32 selectedID = currentID; + for (id =refinedStart; id < refinedEnd; id++) { + if (b[ suffix[id] + searchLength] != currentChar) { + if (currentCount > selectedCount) { + selectedCount = currentCount; + selectedID = currentID; + } + currentID = id; + currentChar = b[ suffix[id] + searchLength]; + currentCount = 0; + } + currentCount ++; + } + if (currentCount > selectedCount) { /* for last */ + selectedCount = currentCount; + selectedID = currentID; + } + + if (selectedCount < minRatio) + break; + refinedStart = selectedID; + refinedEnd = refinedStart + selectedCount; + } + + /* evaluate gain based on new ref */ + start = refinedStart; + pos = suffix[refinedStart]; + end = start; + memset(lengthList, 0, sizeof(lengthList)); + + /* look forward */ + do { + end++; + length = ZDICT_count(b + pos, b + suffix[end]); + if (length >= LLIMIT) length = LLIMIT-1; + lengthList[length]++; + } while (length >=MINMATCHLENGTH); + + /* look backward */ + length = MINMATCHLENGTH; + while ((length >= MINMATCHLENGTH) & (start > 0)) { + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; + } + + /* largest useful length */ + memset(cumulLength, 0, sizeof(cumulLength)); + cumulLength[maxLength-1] = lengthList[maxLength-1]; + for (i=(int)(maxLength-2); i>=0; i--) + cumulLength[i] = cumulLength[i+1] + lengthList[i]; + + for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break; + maxLength = i; + + /* reduce maxLength in case of final into repetitive data */ + { U32 l = (U32)maxLength; + BYTE const c = b[pos + maxLength-1]; + while (b[pos+l-2]==c) l--; + maxLength = l; + } + if (maxLength < MINMATCHLENGTH) return solution; /* skip : no long-enough solution */ + + /* calculate savings */ + savings[5] = 0; + for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) + savings[i] = savings[i-1] + (lengthList[i] * (i-3)); + + DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", + (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); + + solution.pos = (U32)pos; + solution.length = (U32)maxLength; + solution.savings = savings[maxLength]; + + /* mark positions done */ + { U32 id; + for (id=start; id<end; id++) { + U32 p, pEnd; + U32 const testedPos = suffix[id]; + if (testedPos == pos) + length = solution.length; + else { + length = ZDICT_count(b+pos, b+testedPos); + if (length > solution.length) length = solution.length; + } + pEnd = (U32)(testedPos + length); + for (p=testedPos; p<pEnd; p++) + doneMarks[p] = 1; + } } } + + return solution; +} + + +/*! ZDICT_checkMerge + check if dictItem can be merged, do it if possible + @return : id of destination elt, 0 if not merged +*/ +static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) +{ + const U32 tableSize = table->pos; + const U32 eltEnd = elt.pos + elt.length; + + /* tail overlap */ + U32 u; for (u=1; u<tableSize; u++) { + if (u==eltNbToSkip) continue; + if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ + /* append */ + U32 addedLength = table[u].pos - elt.pos; + table[u].length += addedLength; + table[u].pos = elt.pos; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ + elt = table[u]; + /* sort : improve rank */ + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + /* front overlap */ + for (u=1; u<tableSize; u++) { + if (u==eltNbToSkip) continue; + if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ + /* append */ + int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ + table[u].length += addedLength; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + } + /* sort : improve rank */ + elt = table[u]; + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + return 0; +} + + +static void ZDICT_removeDictItem(dictItem* table, U32 id) +{ + /* convention : first element is nb of elts */ + U32 const max = table->pos; + U32 u; + if (!id) return; /* protection, should never happen */ + for (u=id; u<max-1; u++) + table[u] = table[u+1]; + table->pos--; +} + + +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt) +{ + /* merge if possible */ + U32 mergeId = ZDICT_checkMerge(table, elt, 0); + if (mergeId) { + U32 newMerge = 1; + while (newMerge) { + newMerge = ZDICT_checkMerge(table, table[mergeId], mergeId); + if (newMerge) ZDICT_removeDictItem(table, mergeId); + mergeId = newMerge; + } + return; + } + + /* insert */ + { U32 current; + U32 nextElt = table->pos; + if (nextElt >= maxSize) nextElt = maxSize-1; + current = nextElt-1; + while (table[current].savings < elt.savings) { + table[current+1] = table[current]; + current--; + } + table[current+1] = elt; + table->pos = nextElt+1; + } +} + + +static U32 ZDICT_dictSize(const dictItem* dictList) +{ + U32 u, dictSize = 0; + for (u=1; u<dictList[0].pos; u++) + dictSize += dictList[u].length; + return dictSize; +} + + +static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, + const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */ + const size_t* fileSizes, unsigned nbFiles, + U32 minRatio, U32 notificationLevel) +{ + int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0)); + int* const suffix = suffix0+1; + U32* reverseSuffix = (U32*)malloc((bufferSize)*sizeof(*reverseSuffix)); + BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks)); /* +16 for overflow security */ + U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos)); + size_t result = 0; + clock_t displayClock = 0; + clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; + +# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) \ + { displayClock = clock(); DISPLAY(__VA_ARGS__); \ + if (notificationLevel>=4) fflush(stdout); } } + + /* init */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + if (!suffix0 || !reverseSuffix || !doneMarks || !filePos) { + result = ERROR(memory_allocation); + goto _cleanup; + } + if (minRatio < MINRATIO) minRatio = MINRATIO; + memset(doneMarks, 0, bufferSize+16); + + /* limit sample set size (divsufsort limitation)*/ + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); + while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; + + /* sort */ + DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + } + suffix[bufferSize] = (int)bufferSize; /* leads into noise */ + suffix0[0] = (int)bufferSize; /* leads into noise */ + /* build reverse suffix sort */ + { size_t pos; + for (pos=0; pos < bufferSize; pos++) + reverseSuffix[suffix[pos]] = (U32)pos; + /* note filePos tracks borders between samples. + It's not used at this stage, but planned to become useful in a later update */ + filePos[0] = 0; + for (pos=1; pos<nbFiles; pos++) + filePos[pos] = (U32)(filePos[pos-1] + fileSizes[pos-1]); + } + + DISPLAYLEVEL(2, "finding patterns ... \n"); + DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio); + + { U32 cursor; for (cursor=0; cursor < bufferSize; ) { + dictItem solution; + if (doneMarks[cursor]) { cursor++; continue; } + solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel); + if (solution.length==0) { cursor++; continue; } + ZDICT_insertDictItem(dictList, dictListSize, solution); + cursor += solution.length; + DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); + } } + +_cleanup: + free(suffix0); + free(reverseSuffix); + free(doneMarks); + free(filePos); + return result; +} + + +static void ZDICT_fillNoise(void* buffer, size_t length) +{ + unsigned const prime1 = 2654435761U; + unsigned const prime2 = 2246822519U; + unsigned acc = prime1; + size_t p=0;; + for (p=0; p<length; p++) { + acc *= prime2; + ((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21); + } +} + + +typedef struct +{ + ZSTD_CCtx* ref; + ZSTD_CCtx* zc; + void* workPlace; /* must be ZSTD_BLOCKSIZE_ABSOLUTEMAX allocated */ +} EStats_ress_t; + +#define MAXREPOFFSET 1024 + +static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, + U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, + const void* src, size_t srcSize, U32 notificationLevel) +{ + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog); + size_t cSize; + + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + } + cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } + + if (cSize) { /* if == 0; block is not compressible */ + const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc); + + /* literals stats */ + { const BYTE* bytePtr; + for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++) + countLit[*bytePtr]++; + } + + /* seqStats */ + { U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + ZSTD_seqToCodes(seqStorePtr); + + { const BYTE* codePtr = seqStorePtr->ofCode; + U32 u; + for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++; + } + + { const BYTE* codePtr = seqStorePtr->mlCode; + U32 u; + for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++; + } + + { const BYTE* codePtr = seqStorePtr->llCode; + U32 u; + for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++; + } + + if (nbSeq >= 2) { /* rep offsets */ + const seqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offset - 3; + U32 offset2 = seq[1].offset - 3; + if (offset1 >= MAXREPOFFSET) offset1 = 0; + if (offset2 >= MAXREPOFFSET) offset2 = 0; + repOffsets[offset1] += 3; + repOffsets[offset2] += 1; + } } } +} + +/* +static size_t ZDICT_maxSampleSize(const size_t* fileSizes, unsigned nbFiles) +{ + unsigned u; + size_t max=0; + for (u=0; u<nbFiles; u++) + if (max < fileSizes[u]) max = fileSizes[u]; + return max; +} +*/ + +static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles) +{ + size_t total=0; + unsigned u; + for (u=0; u<nbFiles; u++) total += fileSizes[u]; + return total; +} + +typedef struct { U32 offset; U32 count; } offsetCount_t; + +static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val, U32 count) +{ + U32 u; + table[ZSTD_REP_NUM].offset = val; + table[ZSTD_REP_NUM].count = count; + for (u=ZSTD_REP_NUM; u>0; u--) { + offsetCount_t tmp; + if (table[u-1].count >= table[u].count) break; + tmp = table[u-1]; + table[u-1] = table[u]; + table[u] = tmp; + } +} + + +#define OFFCODE_MAX 30 /* only applicable to first block */ +static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, + unsigned compressionLevel, + const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize, + unsigned notificationLevel) +{ + U32 countLit[256]; + HUF_CREATE_STATIC_CTABLE(hufTable, 255); + U32 offcodeCount[OFFCODE_MAX+1]; + short offcodeNCount[OFFCODE_MAX+1]; + U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB)); + U32 matchLengthCount[MaxML+1]; + short matchLengthNCount[MaxML+1]; + U32 litLengthCount[MaxLL+1]; + short litLengthNCount[MaxLL+1]; + U32 repOffset[MAXREPOFFSET]; + offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; + EStats_ress_t esr; + ZSTD_parameters params; + U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; + size_t pos = 0, errorCode; + size_t eSize = 0; + size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); + size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); + BYTE* dstPtr = (BYTE*)dstBuffer; + + /* init */ + esr.ref = ZSTD_createCCtx(); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX); + if (!esr.ref || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } + if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */ + for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1; + for (u=0; u<=MaxML; u++) matchLengthCount[u]=1; + for (u=0; u<=MaxLL; u++) litLengthCount[u]=1; + memset(repOffset, 0, sizeof(repOffset)); + repOffset[1] = repOffset[4] = repOffset[8] = 1; + memset(bestRepOffset, 0, sizeof(bestRepOffset)); + if (compressionLevel==0) compressionLevel=g_compressionLevel_default; + params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); + { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + if (ZSTD_isError(beginResult)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); + goto _cleanup; + } } + + /* collect stats on all files */ + for (u=0; u<nbFiles; u++) { + ZDICT_countEStats(esr, params, + countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset, + (const char*)srcBuffer + pos, fileSizes[u], + notificationLevel); + pos += fileSizes[u]; + } + + /* analyze */ + errorCode = HUF_buildCTable (hufTable, countLit, 255, huffLog); + if (HUF_isError(errorCode)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "HUF_buildCTable error \n"); + goto _cleanup; + } + huffLog = (U32)errorCode; + + /* looking for most common first offsets */ + { U32 offset; + for (offset=1; offset<MAXREPOFFSET; offset++) + ZDICT_insertSortCount(bestRepOffset, offset, repOffset[offset]); + } + /* note : the result of this phase should be used to better appreciate the impact on statistics */ + + total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u]; + errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax); + if (FSE_isError(errorCode)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n"); + goto _cleanup; + } + Offlog = (U32)errorCode; + + total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u]; + errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML); + if (FSE_isError(errorCode)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n"); + goto _cleanup; + } + mlLog = (U32)errorCode; + + total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u]; + errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL); + if (FSE_isError(errorCode)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n"); + goto _cleanup; + } + llLog = (U32)errorCode; + + /* write result to buffer */ + { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); + if (HUF_isError(hhSize)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "HUF_writeCTable error \n"); + goto _cleanup; + } + dstPtr += hhSize; + maxDstSize -= hhSize; + eSize += hhSize; + } + + { size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog); + if (FSE_isError(ohSize)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n"); + goto _cleanup; + } + dstPtr += ohSize; + maxDstSize -= ohSize; + eSize += ohSize; + } + + { size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog); + if (FSE_isError(mhSize)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n"); + goto _cleanup; + } + dstPtr += mhSize; + maxDstSize -= mhSize; + eSize += mhSize; + } + + { size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog); + if (FSE_isError(lhSize)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n"); + goto _cleanup; + } + dstPtr += lhSize; + maxDstSize -= lhSize; + eSize += lhSize; + } + + if (maxDstSize<12) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "not enough space to write RepOffsets \n"); + goto _cleanup; + } +# if 0 + MEM_writeLE32(dstPtr+0, bestRepOffset[0].offset); + MEM_writeLE32(dstPtr+4, bestRepOffset[1].offset); + MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); +#else + /* at this stage, we don't use the result of "most common first offset", + as the impact of statistics is not properly evaluated */ + MEM_writeLE32(dstPtr+0, repStartValue[0]); + MEM_writeLE32(dstPtr+4, repStartValue[1]); + MEM_writeLE32(dstPtr+8, repStartValue[2]); +#endif + //dstPtr += 12; + eSize += 12; + +_cleanup: + ZSTD_freeCCtx(esr.ref); + ZSTD_freeCCtx(esr.zc); + free(esr.workPlace); + + return eSize; +} + + +size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + size_t hSize; + int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; + U32 const notificationLevel = params.notificationLevel; + + /* dictionary header */ + MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; + MEM_writeLE32((char*)dictBuffer+4, dictID); + } + hSize = 8; + + /* entropy tables */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + DISPLAYLEVEL(2, "statistics ... \n"); + { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, + compressionLevel, + samplesBuffer, samplesSizes, nbSamples, + (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, + notificationLevel); + if (ZDICT_isError(eSize)) return eSize; + hSize += eSize; + } + + + if (hSize + dictContentSize < dictBufferCapacity) + memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + return MIN(dictBufferCapacity, hSize+dictContentSize); +} + + +/*! ZDICT_trainFromBuffer_unsafe() : +* Warning : `samplesBuffer` must be followed by noisy guard band. +* @return : size of dictionary, or an error code which can be tested with ZDICT_isError() +*/ +size_t ZDICT_trainFromBuffer_unsafe( + void* dictBuffer, size_t maxDictSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + U32 const dictListSize = MAX(MAX(DICTLISTSIZE_DEFAULT, nbSamples), (U32)(maxDictSize/16)); + dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); + unsigned const selectivity = params.selectivityLevel == 0 ? g_selectivity_default : params.selectivityLevel; + unsigned const minRep = (selectivity > 30) ? MINRATIO : nbSamples >> selectivity; + size_t const targetDictSize = maxDictSize; + size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + size_t dictSize = 0; + U32 const notificationLevel = params.notificationLevel; + + /* checks */ + if (!dictList) return ERROR(memory_allocation); + if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); } + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return 0; } /* not enough source to create dictionary */ + + /* init */ + ZDICT_initDictItem(dictList); + + /* build dictionary */ + ZDICT_trainBuffer(dictList, dictListSize, + samplesBuffer, samplesBuffSize, + samplesSizes, nbSamples, + minRep, notificationLevel); + + /* display best matches */ + if (params.notificationLevel>= 3) { + U32 const nb = MIN(25, dictList[0].pos); + U32 const dictContentSize = ZDICT_dictSize(dictList); + U32 u; + DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize); + DISPLAYLEVEL(3, "list %u best segments \n", nb); + for (u=1; u<=nb; u++) { + U32 pos = dictList[u].pos; + U32 length = dictList[u].length; + U32 printedLength = MIN(40, length); + DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", + u, length, pos, dictList[u].savings); + ZDICT_printHex((const char*)samplesBuffer+pos, printedLength); + DISPLAYLEVEL(3, "| \n"); + } } + + + /* create dictionary */ + { U32 dictContentSize = ZDICT_dictSize(dictList); + if (dictContentSize < targetDictSize/3) { + DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize); + if (minRep > MINRATIO) { + DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); + DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n"); + } + if (samplesBuffSize < 10 * targetDictSize) + DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); + } + + if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { + U32 proposedSelectivity = selectivity-1; + while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } + DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize); + DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity); + DISPLAYLEVEL(2, "! always test dictionary efficiency on samples \n"); + } + + /* limit dictionary size */ + { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */ + U32 currentSize = 0; + U32 n; for (n=1; n<max; n++) { + currentSize += dictList[n].length; + if (currentSize > targetDictSize) { currentSize -= dictList[n].length; break; } + } + dictList->pos = n; + dictContentSize = currentSize; + } + + /* build dict content */ + { U32 u; + BYTE* ptr = (BYTE*)dictBuffer + maxDictSize; + for (u=1; u<dictList->pos; u++) { + U32 l = dictList[u].length; + ptr -= l; + if (ptr<(BYTE*)dictBuffer) { free(dictList); return ERROR(GENERIC); } /* should not happen */ + memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); + } } + + dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize, + samplesBuffer, samplesSizes, nbSamples, + params); + } + + /* clean up */ + free(dictList); + return dictSize; +} + + +/* issue : samplesBuffer need to be followed by a noisy guard band. +* work around : duplicate the buffer, and add the noise */ +size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + size_t result; + void* newBuff; + size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */ + + newBuff = malloc(sBuffSize + NOISELENGTH); + if (!newBuff) return ERROR(memory_allocation); + + memcpy(newBuff, samplesBuffer, sBuffSize); + ZDICT_fillNoise((char*)newBuff + sBuffSize, NOISELENGTH); /* guard band, for end of buffer condition */ + + result = ZDICT_trainFromBuffer_unsafe( + dictBuffer, dictBufferCapacity, + newBuff, samplesSizes, nbSamples, + params); + free(newBuff); + return result; +} + + +size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_trainFromBuffer_advanced(dictBuffer, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} + +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/dictBuilder/zdict.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef DICTBUILDER_H_001 +#define DICTBUILDER_H_001 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*====== Dependencies ======*/ +#include <stddef.h> /* size_t */ + + +/*====== Export for Windows ======*/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) +#else +# define ZDICTLIB_API +#endif + + +/*! ZDICT_trainFromBuffer() : + Train a dictionary from an array of samples. + Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + The resulting dictionary will be saved into `dictBuffer`. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + or an error code, which can be tested with ZDICT_isError(). + Tips : In general, a reasonable dictionary has a size of ~ 100 KB. + It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + In general, it's recommended to provide a few thousands samples, but this can vary a lot. + It's recommended that total size of all samples be about ~x100 times the target size of dictionary. +*/ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + + + +#ifdef ZDICT_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + int compressionLevel; /* 0 means default; target a specific zstd compression level */ + unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ + unsigned reserved[2]; /* reserved space for future parameters */ +} ZDICT_params_t; + + +/*! ZDICT_trainFromBuffer_advanced() : + Same as ZDICT_trainFromBuffer() with control over more parameters. + `parameters` is optional and can be provided with values set to 0 to mean "default". + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`), + or an error code, which can be tested by ZDICT_isError(). + note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0. +*/ +size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*! ZDICT_addEntropyTablesFromBuffer() : + + Given a content-only dictionary (built using any 3rd party algorithm), + add entropy tables computed from an array of samples. + Samples must be stored concatenated in a flat buffer `samplesBuffer`, + supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. + + The input dictionary content must be stored *at the end* of `dictBuffer`. + Its size is `dictContentSize`. + The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*, + starting from its beginning. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`). +*/ +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + + +#endif /* ZDICT_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif + +#endif /* DICTBUILDER_H_001 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd/zstd.h Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,640 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ====== Dependency ======*/ +#include <stddef.h> /* size_t */ + + +/* ====== Export for Windows ======*/ +/* +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + +/******************************************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios + at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and + decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. + Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory. + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit memory management) + - unbounded multiple steps (described as Streaming compression) + The compression ratio achievable on small data can be highly improved using compression with a dictionary in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Fast dictionary API) + + Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. + These APIs shall never be used with a dynamic library. + They are not "stable", their definition may change in the future. Only static linking is allowed. +*********************************************************************************************************/ + +/*------ Version ------*/ +ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */ + +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 1 +#define ZSTD_VERSION_RELEASE 1 + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + + +/*************************************** +* Simple API +***************************************/ +/*! ZSTD_compress() : + Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + @return : compressed size written into `dst` (<= `dstCapacity), + or an error code if it fails (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + `compressedSize` : must be the _exact_ size of a single compressed frame. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + or an errorCode if it fails (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getDecompressedSize() : +* 'src' is the start of a zstd compressed frame. +* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==0`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ + + +/*************************************** +* Explicit memory management +***************************************/ +/*= Compression context +* When compressing many messages / blocks, +* it is recommended to allocate a context just once, and re-use it for each successive compression operation. +* This will make the situation much easier for the system's memory. +* Use one context per thread for parallel execution in multi-threaded environments. */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); + +/*! ZSTD_compressCCtx() : + Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); + +/*= Decompression context */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); + +/*! ZSTD_decompressDCtx() : +* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : +* Compression using a predefined Dictionary (see dictBuilder/zdict.h). +* Note : This function load the dictionary, resulting in significant startup delay. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : +* Decompression using a predefined Dictionary (see dictBuilder/zdict.h). +* Dictionary must be identical to the one used during compression. +* Note : This function load the dictionary, resulting in significant startup delay */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/**************************** +* Fast dictionary API +****************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : +* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. +* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. +* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. +* `dict` can be released after ZSTD_CDict creation */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); + +/*! ZSTD_freeCDict() : +* Function frees memory allocated by ZSTD_createCDict() */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a digested Dictionary. +* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); + +/*! ZSTD_freeDDict() : +* Function frees memory allocated with ZSTD_createDDict() */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a digested Dictionary +* Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, +* since it will play nicer with system's memory, by re-using already allocated memory. +* Use one separate ZSTD_CStream per thread for parallel execution. +* +* Start a new compression by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically update both `pos` fields. +* Note that it may not consume the entire input, in which case `pos < size`, +* and it's up to the caller to present again remaining data. +* @return : a size hint, preferred nb of bytes to use as input for next function call +* (it's just a hint, to help latency a little, any other value will work fine) +* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() ) +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* `output->pos` will be updated. +* Note some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +/*===== Streaming compression functions ======*/ +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* @return : recommended first input size +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* If `output.pos < output.size`, decoder has flushed everything it could. +* @return : 0 when a frame is completely decoded and fully flushed, +* an error code, which can be tested using ZSTD_isError(), +* any other value > 0, which means there is still some work to do to complete the frame. +* The return value is a suggested next input size (just an hint, to help latency). +* *******************************************************************************/ + +/*===== Streaming decompression functions =====*/ +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + + +#ifdef ZSTD_STATIC_LINKING_ONLY + +/**************************************************************************************** + * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ***************************************************************************************/ + +/* --- Constants ---*/ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define ZSTD_WINDOWLOG_MAX_32 25 +#define ZSTD_WINDOWLOG_MAX_64 27 +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MIN 4 +#define ZSTD_TARGETLENGTH_MAX 999 + +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_prefix = 5; +static const size_t ZSTD_frameHeaderSize_min = 6; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ + + +/*--- Advanced types ---*/ +typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */ + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; +} ZSTD_compressionParameters; + +typedef struct { + unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +/*= Custom memory allocation functions */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; + + +/*************************************** +* Advanced compression functions +***************************************/ +/*! ZSTD_estimateCCtxSize() : + * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. + * `frameContentSize` is an optional parameter, provide `0` if unknown */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); + +/*! ZSTD_createCCtx_advanced() : + * Create a ZSTD compression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); + +/*! ZSTD_createCDict_advanced() : + * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_parameters params, ZSTD_customMem customMem); + +/*! ZSTD_sizeof_CDict() : + * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getParams() : +* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. +* All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level and srcSize. +* `srcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : +* Ensure param values remain within authorized range */ +ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : +* optimize params for a given `srcSize` and `dictSize`. +* both values are optional, select `0` if unknown. */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : +* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + + +/*--- Advanced decompression functions ---*/ + +/*! ZSTD_estimateDCtxSize() : + * Gives the potential amount of memory allocated to create a ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); + +/*! ZSTD_sizeof_DCtx() : + * Gives the amount of memory used by a given ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); + +/*! ZSTD_sizeof_DDict() : + * Gives the amount of memory used by a given ZSTD_DDict */ +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + + +/******************************************************************** +* Advanced streaming functions +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */ +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); + + +/*===== Advanced Streaming decompression functions =====*/ +typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */ +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* But it's also a complex one, with many restrictions (documented below). +* Prefer using normal streaming API for an easier experience +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, + or ZSTD_compressBegin_advanced(), for finer parameter control. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. + - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame, + Without last block mark, frames will be considered unfinished (broken) by decoders. + + You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + + +/*- + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame, + such as the minimum rolling buffer size to allocate to decompress data (`windowSize`), + and the dictionary ID used. + (Note : content size is optional, it may not be present. 0 means : content size unknown). + Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information. + As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation. + Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB. + Frame parameters are extracted from the beginning of the compressed frame. + Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes. + @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. + They should preferably be located contiguously, prior to current block. + Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference. + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_decompressContinue() always returns 0. + For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. +*/ + +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameParams; + +/*===== Buffer-less streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +/** + Block functions + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : ZSTD_compressBegin() + + decompression : ZSTD_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - Block size is limited, it must be <= ZSTD_getBlockSizeMax() + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTD_insertBlock() in such a case. +*/ + +#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +/*===== Raw zstd block functions =====*/ +ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ + + +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_235446 */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/python-zstandard/zstd_cffi.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,152 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +"""Python interface to the Zstandard (zstd) compression library.""" + +from __future__ import absolute_import, unicode_literals + +import io + +from _zstd_cffi import ( + ffi, + lib, +) + + +_CSTREAM_IN_SIZE = lib.ZSTD_CStreamInSize() +_CSTREAM_OUT_SIZE = lib.ZSTD_CStreamOutSize() + + +class _ZstdCompressionWriter(object): + def __init__(self, cstream, writer): + self._cstream = cstream + self._writer = writer + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + if not exc_type and not exc_value and not exc_tb: + out_buffer = ffi.new('ZSTD_outBuffer *') + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + while True: + res = lib.ZSTD_endStream(self._cstream, out_buffer) + if lib.ZSTD_isError(res): + raise Exception('error ending compression stream: %s' % lib.ZSTD_getErrorName) + + if out_buffer.pos: + self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + out_buffer.pos = 0 + + if res == 0: + break + + return False + + def write(self, data): + out_buffer = ffi.new('ZSTD_outBuffer *') + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + # TODO can we reuse existing memory? + in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer.src = ffi.new('char[]', data) + in_buffer.size = len(data) + in_buffer.pos = 0 + while in_buffer.pos < in_buffer.size: + res = lib.ZSTD_compressStream(self._cstream, out_buffer, in_buffer) + if lib.ZSTD_isError(res): + raise Exception('zstd compress error: %s' % lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + out_buffer.pos = 0 + + +class ZstdCompressor(object): + def __init__(self, level=3, dict_data=None, compression_params=None): + if dict_data: + raise Exception('dict_data not yet supported') + if compression_params: + raise Exception('compression_params not yet supported') + + self._compression_level = level + + def compress(self, data): + # Just use the stream API for now. + output = io.BytesIO() + with self.write_to(output) as compressor: + compressor.write(data) + return output.getvalue() + + def copy_stream(self, ifh, ofh): + cstream = self._get_cstream() + + in_buffer = ffi.new('ZSTD_inBuffer *') + out_buffer = ffi.new('ZSTD_outBuffer *') + + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + total_read, total_write = 0, 0 + + while True: + data = ifh.read(_CSTREAM_IN_SIZE) + if not data: + break + + total_read += len(data) + + in_buffer.src = ffi.new('char[]', data) + in_buffer.size = len(data) + in_buffer.pos = 0 + + while in_buffer.pos < in_buffer.size: + res = lib.ZSTD_compressStream(cstream, out_buffer, in_buffer) + if lib.ZSTD_isError(res): + raise Exception('zstd compress error: %s' % + lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + total_write = out_buffer.pos + out_buffer.pos = 0 + + # We've finished reading. Flush the compressor. + while True: + res = lib.ZSTD_endStream(cstream, out_buffer) + if lib.ZSTD_isError(res): + raise Exception('error ending compression stream: %s' % + lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + total_write += out_buffer.pos + out_buffer.pos = 0 + + if res == 0: + break + + return total_read, total_write + + def write_to(self, writer): + return _ZstdCompressionWriter(self._get_cstream(), writer) + + def _get_cstream(self): + cstream = lib.ZSTD_createCStream() + cstream = ffi.gc(cstream, lib.ZSTD_freeCStream) + + res = lib.ZSTD_initCStream(cstream, self._compression_level) + if lib.ZSTD_isError(res): + raise Exception('cannot init CStream: %s' % + lib.ZSTD_getErrorName(res)) + + return cstream
--- a/contrib/vim/patchreview.vim Mon Nov 14 22:43:25 2016 +0100 +++ b/contrib/vim/patchreview.vim Sun Nov 27 20:44:52 2016 -0500 @@ -720,7 +720,7 @@ let s:origtabpagenr = tabpagenr() silent! exe 'tabedit ' . StrippedRelativeFilePath if exists('patchcmd') - " modelines in loaded files mess with diff comparision + " modelines in loaded files mess with diff comparison let s:keep_modeline=&modeline let &modeline=0 silent! exe 'vert diffsplit ' . tmpname . '.file'
--- a/hgext/bugzilla.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/bugzilla.py Sun Nov 27 20:44:52 2016 -0500 @@ -571,9 +571,9 @@ self.send_user_agent(h) self.send_content(h, request_body) - # Deal with differences between Python 2.4-2.6 and 2.7. + # Deal with differences between Python 2.6 and 2.7. # In the former h is a HTTP(S). In the latter it's a - # HTTP(S)Connection. Luckily, the 2.4-2.6 implementation of + # HTTP(S)Connection. Luckily, the 2.6 implementation of # HTTP(S) has an underlying HTTP(S)Connection, so extract # that and use it. try:
--- a/hgext/chgserver.py Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,643 +0,0 @@ -# chgserver.py - command server extension for cHg -# -# Copyright 2011 Yuya Nishihara <yuya@tcha.org> -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -"""command server extension for cHg (EXPERIMENTAL) - -'S' channel (read/write) - propagate ui.system() request to client - -'attachio' command - attach client's stdio passed by sendmsg() - -'chdir' command - change current directory - -'getpager' command - checks if pager is enabled and which pager should be executed - -'setenv' command - replace os.environ completely - -'setumask' command - set umask - -'validate' command - reload the config and check if the server is up to date - -Config ------- - -:: - - [chgserver] - idletimeout = 3600 # seconds, after which an idle server will exit - skiphash = False # whether to skip config or env change checks -""" - -from __future__ import absolute_import - -import errno -import hashlib -import inspect -import os -import re -import signal -import struct -import sys -import time - -from mercurial.i18n import _ - -from mercurial import ( - cmdutil, - commands, - commandserver, - dispatch, - error, - extensions, - osutil, - util, -) - -# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for -# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should -# be specifying the version(s) of Mercurial they are tested with, or -# leave the attribute unspecified. -testedwith = 'ships-with-hg-core' - -_log = commandserver.log - -def _hashlist(items): - """return sha1 hexdigest for a list""" - return hashlib.sha1(str(items)).hexdigest() - -# sensitive config sections affecting confighash -_configsections = [ - 'alias', # affects global state commands.table - 'extdiff', # uisetup will register new commands - 'extensions', -] - -# sensitive environment variables affecting confighash -_envre = re.compile(r'''\A(?: - CHGHG - |HG.* - |LANG(?:UAGE)? - |LC_.* - |LD_.* - |PATH - |PYTHON.* - |TERM(?:INFO)? - |TZ - )\Z''', re.X) - -def _confighash(ui): - """return a quick hash for detecting config/env changes - - confighash is the hash of sensitive config items and environment variables. - - for chgserver, it is designed that once confighash changes, the server is - not qualified to serve its client and should redirect the client to a new - server. different from mtimehash, confighash change will not mark the - server outdated and exit since the user can have different configs at the - same time. - """ - sectionitems = [] - for section in _configsections: - sectionitems.append(ui.configitems(section)) - sectionhash = _hashlist(sectionitems) - envitems = [(k, v) for k, v in os.environ.iteritems() if _envre.match(k)] - envhash = _hashlist(sorted(envitems)) - return sectionhash[:6] + envhash[:6] - -def _getmtimepaths(ui): - """get a list of paths that should be checked to detect change - - The list will include: - - extensions (will not cover all files for complex extensions) - - mercurial/__version__.py - - python binary - """ - modules = [m for n, m in extensions.extensions(ui)] - try: - from mercurial import __version__ - modules.append(__version__) - except ImportError: - pass - files = [sys.executable] - for m in modules: - try: - files.append(inspect.getabsfile(m)) - except TypeError: - pass - return sorted(set(files)) - -def _mtimehash(paths): - """return a quick hash for detecting file changes - - mtimehash calls stat on given paths and calculate a hash based on size and - mtime of each file. mtimehash does not read file content because reading is - expensive. therefore it's not 100% reliable for detecting content changes. - it's possible to return different hashes for same file contents. - it's also possible to return a same hash for different file contents for - some carefully crafted situation. - - for chgserver, it is designed that once mtimehash changes, the server is - considered outdated immediately and should no longer provide service. - - mtimehash is not included in confighash because we only know the paths of - extensions after importing them (there is imp.find_module but that faces - race conditions). We need to calculate confighash without importing. - """ - def trystat(path): - try: - st = os.stat(path) - return (st.st_mtime, st.st_size) - except OSError: - # could be ENOENT, EPERM etc. not fatal in any case - pass - return _hashlist(map(trystat, paths))[:12] - -class hashstate(object): - """a structure storing confighash, mtimehash, paths used for mtimehash""" - def __init__(self, confighash, mtimehash, mtimepaths): - self.confighash = confighash - self.mtimehash = mtimehash - self.mtimepaths = mtimepaths - - @staticmethod - def fromui(ui, mtimepaths=None): - if mtimepaths is None: - mtimepaths = _getmtimepaths(ui) - confighash = _confighash(ui) - mtimehash = _mtimehash(mtimepaths) - _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash)) - return hashstate(confighash, mtimehash, mtimepaths) - -# copied from hgext/pager.py:uisetup() -def _setuppagercmd(ui, options, cmd): - if not ui.formatted(): - return - - p = ui.config("pager", "pager", os.environ.get("PAGER")) - usepager = False - always = util.parsebool(options['pager']) - auto = options['pager'] == 'auto' - - if not p: - pass - elif always: - usepager = True - elif not auto: - usepager = False - else: - attended = ['annotate', 'cat', 'diff', 'export', 'glog', 'log', 'qdiff'] - attend = ui.configlist('pager', 'attend', attended) - ignore = ui.configlist('pager', 'ignore') - cmds, _ = cmdutil.findcmd(cmd, commands.table) - - for cmd in cmds: - var = 'attend-%s' % cmd - if ui.config('pager', var): - usepager = ui.configbool('pager', var) - break - if (cmd in attend or - (cmd not in ignore and not attend)): - usepager = True - break - - if usepager: - ui.setconfig('ui', 'formatted', ui.formatted(), 'pager') - ui.setconfig('ui', 'interactive', False, 'pager') - return p - -def _newchgui(srcui, csystem): - class chgui(srcui.__class__): - def __init__(self, src=None): - super(chgui, self).__init__(src) - if src: - self._csystem = getattr(src, '_csystem', csystem) - else: - self._csystem = csystem - - def system(self, cmd, environ=None, cwd=None, onerr=None, - errprefix=None): - # fallback to the original system method if the output needs to be - # captured (to self._buffers), or the output stream is not stdout - # (e.g. stderr, cStringIO), because the chg client is not aware of - # these situations and will behave differently (write to stdout). - if (any(s[1] for s in self._bufferstates) - or not util.safehasattr(self.fout, 'fileno') - or self.fout.fileno() != sys.stdout.fileno()): - return super(chgui, self).system(cmd, environ, cwd, onerr, - errprefix) - # copied from mercurial/util.py:system() - self.flush() - def py2shell(val): - if val is None or val is False: - return '0' - if val is True: - return '1' - return str(val) - env = os.environ.copy() - if environ: - env.update((k, py2shell(v)) for k, v in environ.iteritems()) - env['HG'] = util.hgexecutable() - rc = self._csystem(cmd, env, cwd) - if rc and onerr: - errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]), - util.explainexit(rc)[0]) - if errprefix: - errmsg = '%s: %s' % (errprefix, errmsg) - raise onerr(errmsg) - return rc - - return chgui(srcui) - -def _loadnewui(srcui, args): - newui = srcui.__class__() - for a in ['fin', 'fout', 'ferr', 'environ']: - setattr(newui, a, getattr(srcui, a)) - if util.safehasattr(srcui, '_csystem'): - newui._csystem = srcui._csystem - - # internal config: extensions.chgserver - newui.setconfig('extensions', 'chgserver', - srcui.config('extensions', 'chgserver'), '--config') - - # command line args - args = args[:] - dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args)) - - # stolen from tortoisehg.util.copydynamicconfig() - for section, name, value in srcui.walkconfig(): - source = srcui.configsource(section, name) - if ':' in source or source == '--config': - # path:line or command line - continue - if source == 'none': - # ui.configsource returns 'none' by default - source = '' - newui.setconfig(section, name, value, source) - - # load wd and repo config, copied from dispatch.py - cwds = dispatch._earlygetopt(['--cwd'], args) - cwd = cwds and os.path.realpath(cwds[-1]) or None - rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args) - path, newlui = dispatch._getlocal(newui, rpath, wd=cwd) - - return (newui, newlui) - -class channeledsystem(object): - """Propagate ui.system() request in the following format: - - payload length (unsigned int), - cmd, '\0', - cwd, '\0', - envkey, '=', val, '\0', - ... - envkey, '=', val - - and waits: - - exitcode length (unsigned int), - exitcode (int) - """ - def __init__(self, in_, out, channel): - self.in_ = in_ - self.out = out - self.channel = channel - - def __call__(self, cmd, environ, cwd): - args = [util.quotecommand(cmd), os.path.abspath(cwd or '.')] - args.extend('%s=%s' % (k, v) for k, v in environ.iteritems()) - data = '\0'.join(args) - self.out.write(struct.pack('>cI', self.channel, len(data))) - self.out.write(data) - self.out.flush() - - length = self.in_.read(4) - length, = struct.unpack('>I', length) - if length != 4: - raise error.Abort(_('invalid response')) - rc, = struct.unpack('>i', self.in_.read(4)) - return rc - -_iochannels = [ - # server.ch, ui.fp, mode - ('cin', 'fin', 'rb'), - ('cout', 'fout', 'wb'), - ('cerr', 'ferr', 'wb'), -] - -class chgcmdserver(commandserver.server): - def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress): - super(chgcmdserver, self).__init__( - _newchgui(ui, channeledsystem(fin, fout, 'S')), repo, fin, fout) - self.clientsock = sock - self._oldios = [] # original (self.ch, ui.fp, fd) before "attachio" - self.hashstate = hashstate - self.baseaddress = baseaddress - if hashstate is not None: - self.capabilities = self.capabilities.copy() - self.capabilities['validate'] = chgcmdserver.validate - - def cleanup(self): - super(chgcmdserver, self).cleanup() - # dispatch._runcatch() does not flush outputs if exception is not - # handled by dispatch._dispatch() - self.ui.flush() - self._restoreio() - - def attachio(self): - """Attach to client's stdio passed via unix domain socket; all - channels except cresult will no longer be used - """ - # tell client to sendmsg() with 1-byte payload, which makes it - # distinctive from "attachio\n" command consumed by client.read() - self.clientsock.sendall(struct.pack('>cI', 'I', 1)) - clientfds = osutil.recvfds(self.clientsock.fileno()) - _log('received fds: %r\n' % clientfds) - - ui = self.ui - ui.flush() - first = self._saveio() - for fd, (cn, fn, mode) in zip(clientfds, _iochannels): - assert fd > 0 - fp = getattr(ui, fn) - os.dup2(fd, fp.fileno()) - os.close(fd) - if not first: - continue - # reset buffering mode when client is first attached. as we want - # to see output immediately on pager, the mode stays unchanged - # when client re-attached. ferr is unchanged because it should - # be unbuffered no matter if it is a tty or not. - if fn == 'ferr': - newfp = fp - else: - # make it line buffered explicitly because the default is - # decided on first write(), where fout could be a pager. - if fp.isatty(): - bufsize = 1 # line buffered - else: - bufsize = -1 # system default - newfp = os.fdopen(fp.fileno(), mode, bufsize) - setattr(ui, fn, newfp) - setattr(self, cn, newfp) - - self.cresult.write(struct.pack('>i', len(clientfds))) - - def _saveio(self): - if self._oldios: - return False - ui = self.ui - for cn, fn, _mode in _iochannels: - ch = getattr(self, cn) - fp = getattr(ui, fn) - fd = os.dup(fp.fileno()) - self._oldios.append((ch, fp, fd)) - return True - - def _restoreio(self): - ui = self.ui - for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels): - newfp = getattr(ui, fn) - # close newfp while it's associated with client; otherwise it - # would be closed when newfp is deleted - if newfp is not fp: - newfp.close() - # restore original fd: fp is open again - os.dup2(fd, fp.fileno()) - os.close(fd) - setattr(self, cn, ch) - setattr(ui, fn, fp) - del self._oldios[:] - - def validate(self): - """Reload the config and check if the server is up to date - - Read a list of '\0' separated arguments. - Write a non-empty list of '\0' separated instruction strings or '\0' - if the list is empty. - An instruction string could be either: - - "unlink $path", the client should unlink the path to stop the - outdated server. - - "redirect $path", the client should attempt to connect to $path - first. If it does not work, start a new server. It implies - "reconnect". - - "exit $n", the client should exit directly with code n. - This may happen if we cannot parse the config. - - "reconnect", the client should close the connection and - reconnect. - If neither "reconnect" nor "redirect" is included in the instruction - list, the client can continue with this server after completing all - the instructions. - """ - args = self._readlist() - try: - self.ui, lui = _loadnewui(self.ui, args) - except error.ParseError as inst: - dispatch._formatparse(self.ui.warn, inst) - self.ui.flush() - self.cresult.write('exit 255') - return - newhash = hashstate.fromui(lui, self.hashstate.mtimepaths) - insts = [] - if newhash.mtimehash != self.hashstate.mtimehash: - addr = _hashaddress(self.baseaddress, self.hashstate.confighash) - insts.append('unlink %s' % addr) - # mtimehash is empty if one or more extensions fail to load. - # to be compatible with hg, still serve the client this time. - if self.hashstate.mtimehash: - insts.append('reconnect') - if newhash.confighash != self.hashstate.confighash: - addr = _hashaddress(self.baseaddress, newhash.confighash) - insts.append('redirect %s' % addr) - _log('validate: %s\n' % insts) - self.cresult.write('\0'.join(insts) or '\0') - - def chdir(self): - """Change current directory - - Note that the behavior of --cwd option is bit different from this. - It does not affect --config parameter. - """ - path = self._readstr() - if not path: - return - _log('chdir to %r\n' % path) - os.chdir(path) - - def setumask(self): - """Change umask""" - mask = struct.unpack('>I', self._read(4))[0] - _log('setumask %r\n' % mask) - os.umask(mask) - - def getpager(self): - """Read cmdargs and write pager command to r-channel if enabled - - If pager isn't enabled, this writes '\0' because channeledoutput - does not allow to write empty data. - """ - args = self._readlist() - try: - cmd, _func, args, options, _cmdoptions = dispatch._parse(self.ui, - args) - except (error.Abort, error.AmbiguousCommand, error.CommandError, - error.UnknownCommand): - cmd = None - options = {} - if not cmd or 'pager' not in options: - self.cresult.write('\0') - return - - pagercmd = _setuppagercmd(self.ui, options, cmd) - if pagercmd: - # Python's SIGPIPE is SIG_IGN by default. change to SIG_DFL so - # we can exit if the pipe to the pager is closed - if util.safehasattr(signal, 'SIGPIPE') and \ - signal.getsignal(signal.SIGPIPE) == signal.SIG_IGN: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - self.cresult.write(pagercmd) - else: - self.cresult.write('\0') - - def setenv(self): - """Clear and update os.environ - - Note that not all variables can make an effect on the running process. - """ - l = self._readlist() - try: - newenv = dict(s.split('=', 1) for s in l) - except ValueError: - raise ValueError('unexpected value in setenv request') - _log('setenv: %r\n' % sorted(newenv.keys())) - os.environ.clear() - os.environ.update(newenv) - - capabilities = commandserver.server.capabilities.copy() - capabilities.update({'attachio': attachio, - 'chdir': chdir, - 'getpager': getpager, - 'setenv': setenv, - 'setumask': setumask}) - -def _tempaddress(address): - return '%s.%d.tmp' % (address, os.getpid()) - -def _hashaddress(address, hashstr): - return '%s-%s' % (address, hashstr) - -class chgunixservicehandler(object): - """Set of operations for chg services""" - - pollinterval = 1 # [sec] - - def __init__(self, ui): - self.ui = ui - self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600) - self._lastactive = time.time() - - def bindsocket(self, sock, address): - self._inithashstate(address) - self._checkextensions() - self._bind(sock) - self._createsymlink() - - def _inithashstate(self, address): - self._baseaddress = address - if self.ui.configbool('chgserver', 'skiphash', False): - self._hashstate = None - self._realaddress = address - return - self._hashstate = hashstate.fromui(self.ui) - self._realaddress = _hashaddress(address, self._hashstate.confighash) - - def _checkextensions(self): - if not self._hashstate: - return - if extensions.notloaded(): - # one or more extensions failed to load. mtimehash becomes - # meaningless because we do not know the paths of those extensions. - # set mtimehash to an illegal hash value to invalidate the server. - self._hashstate.mtimehash = '' - - def _bind(self, sock): - # use a unique temp address so we can stat the file and do ownership - # check later - tempaddress = _tempaddress(self._realaddress) - util.bindunixsocket(sock, tempaddress) - self._socketstat = os.stat(tempaddress) - # rename will replace the old socket file if exists atomically. the - # old server will detect ownership change and exit. - util.rename(tempaddress, self._realaddress) - - def _createsymlink(self): - if self._baseaddress == self._realaddress: - return - tempaddress = _tempaddress(self._baseaddress) - os.symlink(os.path.basename(self._realaddress), tempaddress) - util.rename(tempaddress, self._baseaddress) - - def _issocketowner(self): - try: - stat = os.stat(self._realaddress) - return (stat.st_ino == self._socketstat.st_ino and - stat.st_mtime == self._socketstat.st_mtime) - except OSError: - return False - - def unlinksocket(self, address): - if not self._issocketowner(): - return - # it is possible to have a race condition here that we may - # remove another server's socket file. but that's okay - # since that server will detect and exit automatically and - # the client will start a new server on demand. - try: - os.unlink(self._realaddress) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - - def printbanner(self, address): - # no "listening at" message should be printed to simulate hg behavior - pass - - def shouldexit(self): - if not self._issocketowner(): - self.ui.debug('%s is not owned, exiting.\n' % self._realaddress) - return True - if time.time() - self._lastactive > self._idletimeout: - self.ui.debug('being idle too long. exiting.\n') - return True - return False - - def newconnection(self): - self._lastactive = time.time() - - def createcmdserver(self, repo, conn, fin, fout): - return chgcmdserver(self.ui, repo, fin, fout, conn, - self._hashstate, self._baseaddress) - -def chgunixservice(ui, repo, opts): - if repo: - # one chgserver can serve multiple repos. drop repo infomation - ui.setconfig('bundle', 'mainreporoot', '', 'repo') - h = chgunixservicehandler(ui) - return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h) - -def uisetup(ui): - commandserver._servicemap['chgunix'] = chgunixservice - - # CHGINTERNALMARK is temporarily set by chg client to detect if chg will - # start another chg. drop it to avoid possible side effects. - if 'CHGINTERNALMARK' in os.environ: - del os.environ['CHGINTERNALMARK']
--- a/hgext/color.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/color.py Sun Nov 27 20:44:52 2016 -0500 @@ -275,8 +275,8 @@ def modewarn(): # only warn if color.mode was explicitly set and we're in - # an interactive terminal - if mode == realmode and ui.interactive(): + # a formatted terminal + if mode == realmode and ui.formatted(): ui.warn(_('warning: failed to set color mode to %s\n') % mode) if realmode == 'win32': @@ -502,8 +502,8 @@ effects.append(l) effects = ' '.join(effects) if effects: - return '\n'.join([render_effects(s, effects) - for s in msg.split('\n')]) + return '\n'.join([render_effects(line, effects) + for line in msg.split('\n')]) return msg def uisetup(ui): @@ -536,22 +536,50 @@ _("when to colorize (boolean, always, auto, never, or debug)"), _('TYPE'))) -@command('debugcolor', [], 'hg debugcolor') +@command('debugcolor', + [('', 'style', None, _('show all configured styles'))], + 'hg debugcolor') def debugcolor(ui, repo, **opts): + """show available color, effects or style""" + ui.write(('color mode: %s\n') % ui._colormode) + if opts.get('style'): + return _debugdisplaystyle(ui) + else: + return _debugdisplaycolor(ui) + +def _debugdisplaycolor(ui): global _styles - _styles = {} - for effect in _effects.keys(): - _styles[effect] = effect - if _terminfo_params: - for k, v in ui.configitems('color'): - if k.startswith('color.'): - _styles[k] = k[6:] - elif k.startswith('terminfo.'): - _styles[k] = k[9:] - ui.write(('color mode: %s\n') % ui._colormode) - ui.write(_('available colors:\n')) - for colorname, label in _styles.items(): - ui.write(('%s\n') % colorname, label=label) + oldstyle = _styles + try: + _styles = {} + for effect in _effects.keys(): + _styles[effect] = effect + if _terminfo_params: + for k, v in ui.configitems('color'): + if k.startswith('color.'): + _styles[k] = k[6:] + elif k.startswith('terminfo.'): + _styles[k] = k[9:] + ui.write(_('available colors:\n')) + # sort label with a '_' after the other to group '_background' entry. + items = sorted(_styles.items(), + key=lambda i: ('_' in i[0], i[0], i[1])) + for colorname, label in items: + ui.write(('%s\n') % colorname, label=label) + finally: + _styles = oldstyle + +def _debugdisplaystyle(ui): + ui.write(_('available style:\n')) + width = max(len(s) for s in _styles) + for label, effects in sorted(_styles.items()): + ui.write('%s' % label, label=label) + if effects: + # 50 + ui.write(': ') + ui.write(' ' * (max(0, width - len(label)))) + ui.write(', '.join(ui.label(e, e) for e in effects.split())) + ui.write('\n') if os.name != 'nt': w32effects = None
--- a/hgext/convert/common.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/convert/common.py Sun Nov 27 20:44:52 2016 -0500 @@ -454,7 +454,7 @@ if err.errno != errno.ENOENT: raise return - for i, line in enumerate(fp): + for i, line in enumerate(util.iterfile(fp)): line = line.splitlines()[0].rstrip() if not line: # Ignore blank lines
--- a/hgext/convert/convcmd.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/convert/convcmd.py Sun Nov 27 20:44:52 2016 -0500 @@ -201,7 +201,7 @@ m = {} try: fp = open(path, 'r') - for i, line in enumerate(fp): + for i, line in enumerate(util.iterfile(fp)): line = line.splitlines()[0].rstrip() if not line: # Ignore blank lines
--- a/hgext/convert/cvs.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/convert/cvs.py Sun Nov 27 20:44:52 2016 -0500 @@ -15,6 +15,7 @@ from mercurial import ( encoding, error, + pycompat, util, ) @@ -69,7 +70,7 @@ raise error.Abort(_('revision %s is not a patchset number') % self.revs[0]) - d = os.getcwd() + d = pycompat.getcwd() try: os.chdir(self.path) id = None
--- a/hgext/convert/hg.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/convert/hg.py Sun Nov 27 20:44:52 2016 -0500 @@ -352,7 +352,7 @@ p2 = node if self.filemapmode and nparents == 1: - man = self.repo.manifest + man = self.repo.manifestlog._revlog mnode = self.repo.changelog.read(nodemod.bin(p2))[0] closed = 'close' in commit.extra if not closed and not man.cmp(m1node, man.revision(mnode)):
--- a/hgext/convert/subversion.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/convert/subversion.py Sun Nov 27 20:44:52 2016 -0500 @@ -5,7 +5,6 @@ import os import re -import sys import tempfile import xml.dom.minidom @@ -13,6 +12,7 @@ from mercurial import ( encoding, error, + pycompat, scmutil, strutil, util, @@ -164,10 +164,8 @@ raise error.Abort(_('debugsvnlog could not load Subversion python ' 'bindings')) - util.setbinary(sys.stdin) - util.setbinary(sys.stdout) - args = decodeargs(sys.stdin.read()) - get_log_child(sys.stdout, *args) + args = decodeargs(ui.fin.read()) + get_log_child(ui.fout, *args) class logstream(object): """Interruptible revision log iterator.""" @@ -1122,7 +1120,7 @@ self.delexec = [] self.copies = [] self.wc = None - self.cwd = os.getcwd() + self.cwd = pycompat.getcwd() created = False if os.path.isfile(os.path.join(path, '.svn', 'entries')): @@ -1142,7 +1140,8 @@ path = '/' + path path = 'file://' + path - wcpath = os.path.join(os.getcwd(), os.path.basename(path) + '-wc') + wcpath = os.path.join(pycompat.getcwd(), os.path.basename(path) + + '-wc') ui.status(_('initializing svn working copy %r\n') % os.path.basename(wcpath)) self.run0('checkout', path, wcpath)
--- a/hgext/fsmonitor/__init__.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/fsmonitor/__init__.py Sun Nov 27 20:44:52 2016 -0500 @@ -563,7 +563,7 @@ pass class state_update(object): - ''' This context mananger is responsible for dispatching the state-enter + ''' This context manager is responsible for dispatching the state-enter and state-leave signals to the watchman service ''' def __init__(self, repo, node, distance, partial):
--- a/hgext/fsmonitor/state.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/fsmonitor/state.py Sun Nov 27 20:44:52 2016 -0500 @@ -59,6 +59,12 @@ state = file.read().split('\0') # state = hostname\0clock\0ignorehash\0 + list of files, each # followed by a \0 + if len(state) < 3: + self._ui.log( + 'fsmonitor', 'fsmonitor: state file truncated (expected ' + '3 chunks, found %d), nuking state\n', len(state)) + self.invalidate() + return None, None, None diskhostname = state[0] hostname = socket.gethostname() if diskhostname != hostname: @@ -85,12 +91,12 @@ return try: - file = self._opener('fsmonitor.state', 'wb') + file = self._opener('fsmonitor.state', 'wb', atomictemp=True) except (IOError, OSError): self._ui.warn(_("warning: unable to write out fsmonitor state\n")) return - try: + with file: file.write(struct.pack(_versionformat, _version)) file.write(socket.gethostname() + '\0') file.write(clock + '\0') @@ -98,8 +104,6 @@ if notefiles: file.write('\0'.join(notefiles)) file.write('\0') - finally: - file.close() def invalidate(self): try:
--- a/hgext/histedit.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/histedit.py Sun Nov 27 20:44:52 2016 -0500 @@ -173,7 +173,6 @@ import errno import os -import sys from mercurial.i18n import _ from mercurial import ( @@ -991,9 +990,9 @@ return goaleditplan return goalnew -def _readfile(path): +def _readfile(ui, path): if path == '-': - return sys.stdin.read() + return ui.fin.read() else: with open(path, 'rb') as f: return f.read() @@ -1191,7 +1190,7 @@ node.short(state.topmost)) rules = ruleeditor(repo, ui, state.actions, comment) else: - rules = _readfile(rules) + rules = _readfile(ui, rules) actions = parserules(rules, state) ctxs = [repo[act.node] \ for act in state.actions if act.node] @@ -1232,7 +1231,7 @@ actions = [pick(state, r) for r in revs] rules = ruleeditor(repo, ui, actions, comment) else: - rules = _readfile(rules) + rules = _readfile(ui, rules) actions = parserules(rules, state) warnverifyactions(ui, repo, actions, state, ctxs) @@ -1406,12 +1405,12 @@ % node.short(missing[0])) def adjustreplacementsfrommarkers(repo, oldreplacements): - """Adjust replacements from obsolescense markers + """Adjust replacements from obsolescence markers Replacements structure is originally generated based on histedit's state and does not account for changes that are not recorded there. This function fixes that by adding - data read from obsolescense markers""" + data read from obsolescence markers""" if not obsolete.isenabled(repo, obsolete.createmarkersopt): return oldreplacements
--- a/hgext/keyword.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/keyword.py Sun Nov 27 20:44:52 2016 -0500 @@ -737,6 +737,8 @@ return ret def kwfilectx_cmp(orig, self, fctx): + if fctx._customcmp: + return fctx.cmp(self) # keyword affects data size, comparing wdir and filelog size does # not make sense if (fctx._filenode is None and
--- a/hgext/largefiles/lfcommands.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/largefiles/lfcommands.py Sun Nov 27 20:44:52 2016 -0500 @@ -510,18 +510,21 @@ lfdirstate.normal(lfile) update1 = 1 - # copy the state of largefile standin from the repository's + # copy the exec mode of largefile standin from the repository's # dirstate to its state in the lfdirstate. rellfile = lfile relstandin = lfutil.standin(lfile) if wvfs.exists(relstandin): + # exec is decided by the users permissions using mask 0o100 standinexec = wvfs.stat(relstandin).st_mode & 0o100 - st = wvfs.stat(rellfile).st_mode - if standinexec != st & 0o100: - st &= ~0o111 + st = wvfs.stat(rellfile) + mode = st.st_mode + if standinexec != mode & 0o100: + # first remove all X bits, then shift all R bits to X + mode &= ~0o111 if standinexec: - st |= (st >> 2) & 0o111 & ~util.umask - wvfs.chmod(rellfile, st) + mode |= (mode >> 2) & 0o111 & ~util.umask + wvfs.chmod(rellfile, mode) update1 = 1 updated += update1
--- a/hgext/largefiles/proto.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/largefiles/proto.py Sun Nov 27 20:44:52 2016 -0500 @@ -76,7 +76,7 @@ yield '%d\n' % length for chunk in util.filechunkiter(f): yield chunk - return wireproto.streamres(generator()) + return wireproto.streamres(gen=generator()) def statlfile(repo, proto, sha): '''Server command for checking if a largefile is present - returns '2\n' if
--- a/hgext/logtoprocess.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/logtoprocess.py Sun Nov 27 20:44:52 2016 -0500 @@ -27,7 +27,7 @@ would log the warning message and traceback of any failed command dispatch. -Scripts are run asychronously as detached daemon processes; mercurial will +Scripts are run asynchronously as detached daemon processes; mercurial will not ensure that they exit cleanly. """
--- a/hgext/mq.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/mq.py Sun Nov 27 20:44:52 2016 -0500 @@ -79,7 +79,7 @@ from mercurial import ( cmdutil, commands, - dispatch, + dirstateguard, error, extensions, hg, @@ -87,6 +87,7 @@ lock as lockmod, patch as patchmod, phases, + pycompat, registrar, revset, scmutil, @@ -1660,7 +1661,7 @@ # caching against the next repo.status call mm, aa, dd = repo.status(patchparent, top)[:3] changes = repo.changelog.read(top) - man = repo.manifest.read(changes[0]) + man = repo.manifestlog[changes[0]].read() aaa = aa[:] matchfn = scmutil.match(repo[None], pats, opts) # in short mode, we only diff the files included in the @@ -1725,7 +1726,7 @@ dsguard = None try: - dsguard = cmdutil.dirstateguard(repo, 'mq.refresh') + dsguard = dirstateguard.dirstateguard(repo, 'mq.refresh') if diffopts.git or diffopts.upgrade: copies = {} for dst in a: @@ -3523,7 +3524,7 @@ raise error.Abort(_('only a local queue repository ' 'may be initialized')) else: - repopath = cmdutil.findrepo(os.getcwd()) + repopath = cmdutil.findrepo(pycompat.getcwd()) if not repopath: raise error.Abort(_('there is no Mercurial repository here ' '(.hg not found)')) @@ -3588,7 +3589,7 @@ for cmd, entry in cmdtable.iteritems(): cmd = cmdutil.parsealiases(cmd)[0] func = entry[0] - if dispatch._cmdattr(ui, cmd, func, 'norepo'): + if func.norepo: continue entry = extensions.wrapcommand(cmdtable, cmd, mqcommand) entry[1].extend(mqopt)
--- a/hgext/pager.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/pager.py Sun Nov 27 20:44:52 2016 -0500 @@ -84,19 +84,19 @@ def _runpager(ui, p): pager = subprocess.Popen(p, shell=True, bufsize=-1, close_fds=util.closefds, stdin=subprocess.PIPE, - stdout=sys.stdout, stderr=sys.stderr) + stdout=util.stdout, stderr=util.stderr) # back up original file objects and descriptors olduifout = ui.fout - oldstdout = sys.stdout - stdoutfd = os.dup(sys.stdout.fileno()) - stderrfd = os.dup(sys.stderr.fileno()) + oldstdout = util.stdout + stdoutfd = os.dup(util.stdout.fileno()) + stderrfd = os.dup(util.stderr.fileno()) # create new line-buffered stdout so that output can show up immediately - ui.fout = sys.stdout = newstdout = os.fdopen(sys.stdout.fileno(), 'wb', 1) - os.dup2(pager.stdin.fileno(), sys.stdout.fileno()) - if ui._isatty(sys.stderr): - os.dup2(pager.stdin.fileno(), sys.stderr.fileno()) + ui.fout = util.stdout = newstdout = os.fdopen(util.stdout.fileno(), 'wb', 1) + os.dup2(pager.stdin.fileno(), util.stdout.fileno()) + if ui._isatty(util.stderr): + os.dup2(pager.stdin.fileno(), util.stderr.fileno()) @atexit.register def killpager(): @@ -104,13 +104,13 @@ signal.signal(signal.SIGINT, signal.SIG_IGN) pager.stdin.close() ui.fout = olduifout - sys.stdout = oldstdout + util.stdout = oldstdout # close new stdout while it's associated with pager; otherwise stdout # fd would be closed when newstdout is deleted newstdout.close() # restore original fds: stdout is open again - os.dup2(stdoutfd, sys.stdout.fileno()) - os.dup2(stderrfd, sys.stderr.fileno()) + os.dup2(stdoutfd, util.stdout.fileno()) + os.dup2(stderrfd, util.stderr.fileno()) pager.wait() def uisetup(ui):
--- a/hgext/patchbomb.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/patchbomb.py Sun Nov 27 20:44:52 2016 -0500 @@ -166,7 +166,7 @@ while patchlines and not patchlines[0].strip(): patchlines.pop(0) - ds = patch.diffstat(patchlines, git=opts.get('git')) + ds = patch.diffstat(patchlines) if opts.get('diffstat'): body += ds + '\n\n'
--- a/hgext/rebase.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/rebase.py Sun Nov 27 20:44:52 2016 -0500 @@ -32,11 +32,13 @@ commands, copies, destutil, + dirstateguard, error, extensions, hg, lock, - merge, + merge as mergemod, + mergeutil, obsolete, patch, phases, @@ -482,19 +484,20 @@ ui.note(_("update back to initial working directory parent\n")) hg.updaterepo(repo, newwd, False) + if self.currentbookmarks: + with repo.transaction('bookmark') as tr: + updatebookmarks(repo, targetnode, nstate, + self.currentbookmarks, tr) + if self.activebookmark not in repo._bookmarks: + # active bookmark was divergent one and has been deleted + self.activebookmark = None + if not self.keepf: collapsedas = None if self.collapsef: collapsedas = newnode clearrebased(ui, repo, self.state, self.skipped, collapsedas) - with repo.transaction('bookmark') as tr: - if self.currentbookmarks: - updatebookmarks(repo, targetnode, nstate, - self.currentbookmarks, tr) - if self.activebookmark not in repo._bookmarks: - # active bookmark was divergent one and has been deleted - self.activebookmark = None clearstatus(repo) clearcollapsemsg(repo) @@ -661,6 +664,9 @@ _('abort and continue do not allow specifying revisions')) if abortf and opts.get('tool', False): ui.warn(_('tool option will be ignored\n')) + if contf: + ms = mergemod.mergestate.read(repo) + mergeutil.checkunresolved(ms) retcode = rbsrt._prepareabortorcontinue(abortf) if retcode is not None: @@ -786,7 +792,7 @@ '''Commit the wd changes with parents p1 and p2. Reuse commit info from rev but also store useful information in extra. Return node of committed revision.''' - dsguard = cmdutil.dirstateguard(repo, 'rebase') + dsguard = dirstateguard.dirstateguard(repo, 'rebase') try: repo.setparents(repo[p1].node(), repo[p2].node()) ctx = repo[rev] @@ -823,7 +829,7 @@ # Update to target and merge it with local if repo['.'].rev() != p1: repo.ui.debug(" update to %d:%s\n" % (p1, repo[p1])) - merge.update(repo, p1, False, True) + mergemod.update(repo, p1, False, True) else: repo.ui.debug(" already in target\n") repo.dirstate.write(repo.currenttransaction()) @@ -832,8 +838,8 @@ repo.ui.debug(" detach base %d:%s\n" % (base, repo[base])) # When collapsing in-place, the parent is the common ancestor, we # have to allow merging with it. - stats = merge.update(repo, rev, True, True, base, collapse, - labels=['dest', 'source']) + stats = mergemod.update(repo, rev, True, True, base, collapse, + labels=['dest', 'source']) if collapse: copies.duplicatecopies(repo, rev, target) else: @@ -1150,7 +1156,7 @@ # Update away from the rebase if necessary if shouldupdate or needupdate(repo, state): - merge.update(repo, originalwd, False, True) + mergemod.update(repo, originalwd, False, True) # Strip from the first rebased revision if rebased: @@ -1382,7 +1388,7 @@ """return a mapping obsolete => successor for all obsolete nodes to be rebased that have a successors in the destination - obsolete => None entries in the mapping indicate nodes with no succesor""" + obsolete => None entries in the mapping indicate nodes with no successor""" obsoletenotrebased = {} # Build a mapping successor => obsolete nodes for the obsolete
--- a/hgext/shelve.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/shelve.py Sun Nov 27 20:44:52 2016 -0500 @@ -62,6 +62,11 @@ backupdir = 'shelve-backup' shelvedir = 'shelved' +shelvefileextensions = ['hg', 'patch'] + +# we never need the user, so we use a +# generic user for all shelve operations +shelveuser = 'shelve@localhost' class shelvedfile(object): """Helper for the file storing a single shelve @@ -154,6 +159,8 @@ """ _version = 1 _filename = 'shelvedstate' + _keep = 'keep' + _nokeep = 'nokeep' @classmethod def load(cls, repo): @@ -170,6 +177,7 @@ parents = [nodemod.bin(h) for h in fp.readline().split()] stripnodes = [nodemod.bin(h) for h in fp.readline().split()] branchtorestore = fp.readline().strip() + keep = fp.readline().strip() == cls._keep except (ValueError, TypeError) as err: raise error.CorruptedState(str(err)) finally: @@ -183,6 +191,7 @@ obj.parents = parents obj.stripnodes = stripnodes obj.branchtorestore = branchtorestore + obj.keep = keep except error.RepoLookupError as err: raise error.CorruptedState(str(err)) @@ -190,7 +199,7 @@ @classmethod def save(cls, repo, name, originalwctx, pendingctx, stripnodes, - branchtorestore): + branchtorestore, keep=False): fp = repo.vfs(cls._filename, 'wb') fp.write('%i\n' % cls._version) fp.write('%s\n' % name) @@ -201,6 +210,7 @@ fp.write('%s\n' % ' '.join([nodemod.hex(n) for n in stripnodes])) fp.write('%s\n' % branchtorestore) + fp.write('%s\n' % (cls._keep if keep else cls._nokeep)) fp.close() @classmethod @@ -221,7 +231,7 @@ # keep it, because timestamp can't decide exact order of backups continue base = f[:-3] - for ext in 'hg patch'.split(): + for ext in shelvefileextensions: try: vfs.unlink(base + '.' + ext) except OSError as err: @@ -242,24 +252,105 @@ cmdutil.checkunfinished(repo) return _docreatecmd(ui, repo, pats, opts) -def _docreatecmd(ui, repo, pats, opts): - def mutableancestors(ctx): - """return all mutable ancestors for ctx (included) +def getshelvename(repo, parent, opts): + """Decide on the name this shelve is going to have""" + def gennames(): + yield label + for i in xrange(1, 100): + yield '%s-%02d' % (label, i) + name = opts.get('name') + label = repo._activebookmark or parent.branch() or 'default' + # slashes aren't allowed in filenames, therefore we rename it + label = label.replace('/', '_') + + if name: + if shelvedfile(repo, name, 'hg').exists(): + e = _("a shelved change named '%s' already exists") % name + raise error.Abort(e) + else: + for n in gennames(): + if not shelvedfile(repo, n, 'hg').exists(): + name = n + break + else: + raise error.Abort(_("too many shelved changes named '%s'") % label) + + # ensure we are not creating a subdirectory or a hidden file + if '/' in name or '\\' in name: + raise error.Abort(_('shelved change names may not contain slashes')) + if name.startswith('.'): + raise error.Abort(_("shelved change names may not start with '.'")) + return name + +def mutableancestors(ctx): + """return all mutable ancestors for ctx (included) + + Much faster than the revset ancestors(ctx) & draft()""" + seen = set([nodemod.nullrev]) + visit = collections.deque() + visit.append(ctx) + while visit: + ctx = visit.popleft() + yield ctx.node() + for parent in ctx.parents(): + rev = parent.rev() + if rev not in seen: + seen.add(rev) + if parent.mutable(): + visit.append(parent) - Much faster than the revset ancestors(ctx) & draft()""" - seen = set([nodemod.nullrev]) - visit = collections.deque() - visit.append(ctx) - while visit: - ctx = visit.popleft() - yield ctx.node() - for parent in ctx.parents(): - rev = parent.rev() - if rev not in seen: - seen.add(rev) - if parent.mutable(): - visit.append(parent) +def getcommitfunc(extra, interactive, editor=False): + def commitfunc(ui, repo, message, match, opts): + hasmq = util.safehasattr(repo, 'mq') + if hasmq: + saved, repo.mq.checkapplied = repo.mq.checkapplied, False + backup = repo.ui.backupconfig('phases', 'new-commit') + try: + repo.ui.setconfig('phases', 'new-commit', phases.secret) + editor_ = False + if editor: + editor_ = cmdutil.getcommiteditor(editform='shelve.shelve', + **opts) + return repo.commit(message, shelveuser, opts.get('date'), match, + editor=editor_, extra=extra) + finally: + repo.ui.restoreconfig(backup) + if hasmq: + repo.mq.checkapplied = saved + + def interactivecommitfunc(ui, repo, *pats, **opts): + match = scmutil.match(repo['.'], pats, {}) + message = opts['message'] + return commitfunc(ui, repo, message, match, opts) + + return interactivecommitfunc if interactive else commitfunc +def _nothingtoshelvemessaging(ui, repo, pats, opts): + stat = repo.status(match=scmutil.match(repo[None], pats, opts)) + if stat.deleted: + ui.status(_("nothing changed (%d missing files, see " + "'hg status')\n") % len(stat.deleted)) + else: + ui.status(_("nothing changed\n")) + +def _shelvecreatedcommit(repo, node, name): + bases = list(mutableancestors(repo[node])) + shelvedfile(repo, name, 'hg').writebundle(bases, node) + cmdutil.export(repo, [node], + fp=shelvedfile(repo, name, 'patch').opener('wb'), + opts=mdiff.diffopts(git=True)) + +def _includeunknownfiles(repo, pats, opts, extra): + s = repo.status(match=scmutil.match(repo[None], pats, opts), + unknown=True) + if s.unknown: + extra['shelve_unknown'] = '\0'.join(s.unknown) + repo[None].add(s.unknown) + +def _finishshelve(repo): + _aborttransaction(repo) + +def _docreatecmd(ui, repo, pats, opts): wctx = repo[None] parents = wctx.parents() if len(parents) > 1: @@ -267,18 +358,6 @@ parent = parents[0] origbranch = wctx.branch() - # we never need the user, so we use a generic user for all shelve operations - user = 'shelve@localhost' - label = repo._activebookmark or parent.branch() or 'default' - - # slashes aren't allowed in filenames, therefore we rename it - label = label.replace('/', '_') - - def gennames(): - yield label - for i in xrange(1, 100): - yield '%s-%02d' % (label, i) - if parent.node() != nodemod.nullid: desc = "changes to: %s" % parent.description().split('\n', 1)[0] else: @@ -287,8 +366,6 @@ if not opts.get('message'): opts['message'] = desc - name = opts.get('name') - lock = tr = None try: lock = repo.lock() @@ -297,81 +374,31 @@ # pull races. ensure we don't print the abort message to stderr. tr = repo.transaction('commit', report=lambda x: None) - if name: - if shelvedfile(repo, name, 'hg').exists(): - raise error.Abort(_("a shelved change named '%s' already exists" - ) % name) - else: - for n in gennames(): - if not shelvedfile(repo, n, 'hg').exists(): - name = n - break - else: - raise error.Abort(_("too many shelved changes named '%s'") % - label) - - # ensure we are not creating a subdirectory or a hidden file - if '/' in name or '\\' in name: - raise error.Abort(_('shelved change names may not contain slashes')) - if name.startswith('.'): - raise error.Abort(_("shelved change names may not start with '.'")) interactive = opts.get('interactive', False) includeunknown = (opts.get('unknown', False) and not opts.get('addremove', False)) - extra={} + name = getshelvename(repo, parent, opts) + extra = {} if includeunknown: - s = repo.status(match=scmutil.match(repo[None], pats, opts), - unknown=True) - if s.unknown: - extra['shelve_unknown'] = '\0'.join(s.unknown) - repo[None].add(s.unknown) + _includeunknownfiles(repo, pats, opts, extra) if _iswctxonnewbranch(repo) and not _isbareshelve(pats, opts): # In non-bare shelve we don't store newly created branch # at bundled commit repo.dirstate.setbranch(repo['.'].branch()) - def commitfunc(ui, repo, message, match, opts): - hasmq = util.safehasattr(repo, 'mq') - if hasmq: - saved, repo.mq.checkapplied = repo.mq.checkapplied, False - backup = repo.ui.backupconfig('phases', 'new-commit') - try: - repo.ui. setconfig('phases', 'new-commit', phases.secret) - editor = cmdutil.getcommiteditor(editform='shelve.shelve', - **opts) - return repo.commit(message, user, opts.get('date'), match, - editor=editor, extra=extra) - finally: - repo.ui.restoreconfig(backup) - if hasmq: - repo.mq.checkapplied = saved - - def interactivecommitfunc(ui, repo, *pats, **opts): - match = scmutil.match(repo['.'], pats, {}) - message = opts['message'] - return commitfunc(ui, repo, message, match, opts) + commitfunc = getcommitfunc(extra, interactive, editor=True) if not interactive: node = cmdutil.commit(ui, repo, commitfunc, pats, opts) else: - node = cmdutil.dorecord(ui, repo, interactivecommitfunc, None, + node = cmdutil.dorecord(ui, repo, commitfunc, None, False, cmdutil.recordfilter, *pats, **opts) if not node: - stat = repo.status(match=scmutil.match(repo[None], pats, opts)) - if stat.deleted: - ui.status(_("nothing changed (%d missing files, see " - "'hg status')\n") % len(stat.deleted)) - else: - ui.status(_("nothing changed\n")) + _nothingtoshelvemessaging(ui, repo, pats, opts) return 1 - bases = list(mutableancestors(repo[node])) - shelvedfile(repo, name, 'hg').writebundle(bases, node) - cmdutil.export(repo, [node], - fp=shelvedfile(repo, name, 'patch').opener('wb'), - opts=mdiff.diffopts(git=True)) - + _shelvecreatedcommit(repo, node, name) if ui.formatted(): desc = util.ellipsis(desc, ui.termwidth()) @@ -380,7 +407,7 @@ if origbranch != repo['.'].branch() and not _isbareshelve(pats, opts): repo.dirstate.setbranch(origbranch) - _aborttransaction(repo) + _finishshelve(repo) finally: lockmod.release(tr, lock) @@ -399,7 +426,7 @@ with repo.wlock(): for (name, _type) in repo.vfs.readdir(shelvedir): suffix = name.rsplit('.', 1)[-1] - if suffix in ('hg', 'patch'): + if suffix in shelvefileextensions: shelvedfile(repo, name).movetobackup() cleanupoldbackups(repo) @@ -410,8 +437,15 @@ with repo.wlock(): try: for name in pats: - for suffix in 'hg patch'.split(): - shelvedfile(repo, name, suffix).movetobackup() + for suffix in shelvefileextensions: + shfile = shelvedfile(repo, name, suffix) + # patch file is necessary, as it should + # be present for any kind of shelve, + # but the .hg file is optional as in future we + # will add obsolete shelve with does not create a + # bundle + if shfile.exists() or suffix == 'patch': + shfile.movetobackup() cleanupoldbackups(repo) except OSError as err: if err.errno != errno.ENOENT: @@ -476,8 +510,7 @@ for chunk, label in patch.difflabel(iter, difflines): ui.write(chunk, label=label) if opts['stat']: - for chunk, label in patch.diffstatui(difflines, width=width, - git=True): + for chunk, label in patch.diffstatui(difflines, width=width): ui.write(chunk, label=label) def singlepatchcmds(ui, repo, pats, opts, subcommand): @@ -557,8 +590,10 @@ def unshelvecleanup(ui, repo, name, opts): """remove related files after an unshelve""" if not opts.get('keep'): - for filetype in 'hg patch'.split(): - shelvedfile(repo, name, filetype).movetobackup() + for filetype in shelvefileextensions: + shfile = shelvedfile(repo, name, filetype) + if shfile.exists(): + shfile.movetobackup() cleanupoldbackups(repo) def unshelvecontinue(ui, repo, state, opts): @@ -600,6 +635,91 @@ unshelvecleanup(ui, repo, state.name, opts) ui.status(_("unshelve of '%s' complete\n") % state.name) +def _commitworkingcopychanges(ui, repo, opts, tmpwctx): + """Temporarily commit working copy changes before moving unshelve commit""" + # Store pending changes in a commit and remember added in case a shelve + # contains unknown files that are part of the pending change + s = repo.status() + addedbefore = frozenset(s.added) + if not (s.modified or s.added or s.removed or s.deleted): + return tmpwctx, addedbefore + ui.status(_("temporarily committing pending changes " + "(restore with 'hg unshelve --abort')\n")) + commitfunc = getcommitfunc(extra=None, interactive=False, + editor=False) + tempopts = {} + tempopts['message'] = "pending changes temporary commit" + tempopts['date'] = opts.get('date') + ui.quiet = True + node = cmdutil.commit(ui, repo, commitfunc, [], tempopts) + tmpwctx = repo[node] + return tmpwctx, addedbefore + +def _unshelverestorecommit(ui, repo, basename, oldquiet): + """Recreate commit in the repository during the unshelve""" + ui.quiet = True + shelvedfile(repo, basename, 'hg').applybundle() + shelvectx = repo['tip'] + ui.quiet = oldquiet + return repo, shelvectx + +def _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev, basename, pctx, + tmpwctx, shelvectx, branchtorestore): + """Rebase restored commit from its original location to a destination""" + # If the shelve is not immediately on top of the commit + # we'll be merging with, rebase it to be on top. + if tmpwctx.node() == shelvectx.parents()[0].node(): + return shelvectx + + ui.status(_('rebasing shelved changes\n')) + try: + rebase.rebase(ui, repo, **{ + 'rev': [shelvectx.rev()], + 'dest': str(tmpwctx.rev()), + 'keep': True, + 'tool': opts.get('tool', ''), + }) + except error.InterventionRequired: + tr.close() + + stripnodes = [repo.changelog.node(rev) + for rev in xrange(oldtiprev, len(repo))] + shelvedstate.save(repo, basename, pctx, tmpwctx, stripnodes, + branchtorestore, opts.get('keep')) + + util.rename(repo.join('rebasestate'), + repo.join('unshelverebasestate')) + raise error.InterventionRequired( + _("unresolved conflicts (see 'hg resolve', then " + "'hg unshelve --continue')")) + + # refresh ctx after rebase completes + shelvectx = repo['tip'] + + if not shelvectx in tmpwctx.children(): + # rebase was a no-op, so it produced no child commit + shelvectx = tmpwctx + return shelvectx + +def _forgetunknownfiles(repo, shelvectx, addedbefore): + # Forget any files that were unknown before the shelve, unknown before + # unshelve started, but are now added. + shelveunknown = shelvectx.extra().get('shelve_unknown') + if not shelveunknown: + return + shelveunknown = frozenset(shelveunknown.split('\0')) + addedafter = frozenset(repo.status().added) + toforget = (addedafter & shelveunknown) - addedbefore + repo[None].forget(toforget) + +def _finishunshelve(repo, oldtiprev, tr): + # The transaction aborting will strip all the commits for us, + # but it doesn't update the inmemory structures, so addchangegroup + # hooks still fire and try to operate on the missing commits. + # Clean up manually to prevent this. + repo.unfiltered().changelog.strip(oldtiprev, tr) + _aborttransaction(repo) + @command('unshelve', [('a', 'abort', None, _('abort an incomplete unshelve operation')), @@ -667,6 +787,8 @@ try: state = shelvedstate.load(repo) + if opts.get('keep') is None: + opts['keep'] = state.keep except IOError as err: if err.errno != errno.ENOENT: raise @@ -722,101 +844,25 @@ # and shelvectx is the unshelved changes. Then we merge it all down # to the original pctx. - # Store pending changes in a commit and remember added in case a shelve - # contains unknown files that are part of the pending change - s = repo.status() - addedbefore = frozenset(s.added) - if s.modified or s.added or s.removed or s.deleted: - ui.status(_("temporarily committing pending changes " - "(restore with 'hg unshelve --abort')\n")) - def commitfunc(ui, repo, message, match, opts): - hasmq = util.safehasattr(repo, 'mq') - if hasmq: - saved, repo.mq.checkapplied = repo.mq.checkapplied, False + tmpwctx, addedbefore = _commitworkingcopychanges(ui, repo, opts, + tmpwctx) - backup = repo.ui.backupconfig('phases', 'new-commit') - try: - repo.ui.setconfig('phases', 'new-commit', phases.secret) - return repo.commit(message, 'shelve@localhost', - opts.get('date'), match) - finally: - repo.ui.restoreconfig(backup) - if hasmq: - repo.mq.checkapplied = saved - - tempopts = {} - tempopts['message'] = "pending changes temporary commit" - tempopts['date'] = opts.get('date') - ui.quiet = True - node = cmdutil.commit(ui, repo, commitfunc, [], tempopts) - tmpwctx = repo[node] - - ui.quiet = True - shelvedfile(repo, basename, 'hg').applybundle() - - ui.quiet = oldquiet - - shelvectx = repo['tip'] + repo, shelvectx = _unshelverestorecommit(ui, repo, basename, oldquiet) branchtorestore = '' if shelvectx.branch() != shelvectx.p1().branch(): branchtorestore = shelvectx.branch() - # If the shelve is not immediately on top of the commit - # we'll be merging with, rebase it to be on top. - if tmpwctx.node() != shelvectx.parents()[0].node(): - ui.status(_('rebasing shelved changes\n')) - try: - rebase.rebase(ui, repo, **{ - 'rev' : [shelvectx.rev()], - 'dest' : str(tmpwctx.rev()), - 'keep' : True, - 'tool' : opts.get('tool', ''), - }) - except error.InterventionRequired: - tr.close() - - stripnodes = [repo.changelog.node(rev) - for rev in xrange(oldtiprev, len(repo))] - shelvedstate.save(repo, basename, pctx, tmpwctx, stripnodes, - branchtorestore) - - util.rename(repo.join('rebasestate'), - repo.join('unshelverebasestate')) - raise error.InterventionRequired( - _("unresolved conflicts (see 'hg resolve', then " - "'hg unshelve --continue')")) - - # refresh ctx after rebase completes - shelvectx = repo['tip'] - - if not shelvectx in tmpwctx.children(): - # rebase was a no-op, so it produced no child commit - shelvectx = tmpwctx - + shelvectx = _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev, + basename, pctx, tmpwctx, shelvectx, + branchtorestore) mergefiles(ui, repo, pctx, shelvectx) restorebranch(ui, repo, branchtorestore) - - # Forget any files that were unknown before the shelve, unknown before - # unshelve started, but are now added. - shelveunknown = shelvectx.extra().get('shelve_unknown') - if shelveunknown: - shelveunknown = frozenset(shelveunknown.split('\0')) - addedafter = frozenset(repo.status().added) - toforget = (addedafter & shelveunknown) - addedbefore - repo[None].forget(toforget) + _forgetunknownfiles(repo, shelvectx, addedbefore) shelvedstate.clear(repo) - - # The transaction aborting will strip all the commits for us, - # but it doesn't update the inmemory structures, so addchangegroup - # hooks still fire and try to operate on the missing commits. - # Clean up manually to prevent this. - repo.unfiltered().changelog.strip(oldtiprev, tr) - + _finishunshelve(repo, oldtiprev, tr) unshelvecleanup(ui, repo, basename, opts) - - _aborttransaction(repo) finally: ui.quiet = oldquiet if tr:
--- a/hgext/win32mbcs.py Mon Nov 14 22:43:25 2016 +0100 +++ b/hgext/win32mbcs.py Sun Nov 27 20:44:52 2016 -0500 @@ -138,10 +138,7 @@ func = getattr(module, name) def f(*args, **kwds): return wrapper(func, args, kwds) - try: - f.__name__ = func.__name__ # fails with Python 2.3 - except Exception: - pass + f.__name__ = func.__name__ setattr(module, name, f) # List of functions to be wrapped.
--- a/i18n/ja.po Mon Nov 14 22:43:25 2016 +0100 +++ b/i18n/ja.po Sun Nov 27 20:44:52 2016 -0500 @@ -6747,7 +6747,7 @@ msgstr "" msgid "" -"Scripts are run asychronously as detached daemon processes; mercurial will\n" +"Scripts are run asynchronously as detached daemon processes; mercurial will\n" "not ensure that they exit cleanly." msgstr "" @@ -10366,7 +10366,7 @@ msgstr "shelve 状態管理ファイルが破損しています" msgid "please run hg unshelve --abort to abort unshelve operation" -msgstr "'hg unshelve --abort' を実施して unshleve 操作を中断してください" +msgstr "'hg unshelve --abort' を実施して unshelve 操作を中断してください" msgid "" "could not read shelved state file, your working copy may be in an unexpected " @@ -14200,7 +14200,7 @@ msgstr "トランザクション実施途中は廃止マーカを破棄できません" #, python-format -msgid "deleted %i obsolescense markers\n" +msgid "deleted %i obsolescence markers\n" msgstr "%i 個の廃止マーカを破棄\n" #, python-format @@ -32665,8 +32665,8 @@ msgid "revset expects one or more arguments" msgstr "revset の引数は1個以上です" -msgid ":rstdoc(text, style): Format ReStructuredText." -msgstr ":rstdoc(text, style): 出力を ReStructuredText として整形します。" +msgid ":rstdoc(text, style): Format reStructuredText." +msgstr ":rstdoc(text, style): 出力を reStructuredText として整形します。" #. i18n: "rstdoc" is a keyword msgid "rstdoc expects two arguments"
--- a/i18n/pt_BR.po Mon Nov 14 22:43:25 2016 +0100 +++ b/i18n/pt_BR.po Sun Nov 27 20:44:52 2016 -0500 @@ -6761,7 +6761,7 @@ "falhasse." msgid "" -"Scripts are run asychronously as detached daemon processes; mercurial will\n" +"Scripts are run asynchronously as detached daemon processes; mercurial will\n" "not ensure that they exit cleanly." msgstr "" "Os scripts são executados assincronamente como processos desanexados;\n" @@ -14438,7 +14438,7 @@ "não é possível apagar marcações de obsolescência durante uma transação." #, python-format -msgid "deleted %i obsolescense markers\n" +msgid "deleted %i obsolescence markers\n" msgstr "%i marcações de obsolescência apagadas\n" #, python-format @@ -33523,8 +33523,8 @@ msgid "revset expects one or more arguments" msgstr "revset espera um ou mais argumentos" -msgid ":rstdoc(text, style): Format ReStructuredText." -msgstr ":rstdoc(texto, estilo): Formata ReStructuredText." +msgid ":rstdoc(text, style): Format reStructuredText." +msgstr ":rstdoc(texto, estilo): Formata reStructuredText." #. i18n: "rstdoc" is a keyword msgid "rstdoc expects two arguments"
--- a/mercurial/archival.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/archival.py Sun Nov 27 20:44:52 2016 -0500 @@ -141,7 +141,7 @@ self.mtime = mtime self.fileobj = None - def taropen(name, mode, fileobj=None): + def taropen(mode, name='', fileobj=None): if kind == 'gz': mode = mode[0] if not fileobj: @@ -155,10 +155,9 @@ return tarfile.open(name, mode + kind, fileobj) if isinstance(dest, str): - self.z = taropen(dest, mode='w:') + self.z = taropen('w:', name=dest) else: - # Python 2.5-2.5.1 have a regression that requires a name arg - self.z = taropen(name='', mode='w|', fileobj=dest) + self.z = taropen('w|', fileobj=dest) def addfile(self, name, mode, islink, data): i = tarfile.TarInfo(name)
--- a/mercurial/bdiff.c Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/bdiff.c Sun Nov 27 20:44:52 2016 -0500 @@ -17,6 +17,10 @@ #include "bitmanipulation.h" #include "bdiff.h" +/* Hash implementation from diffutils */ +#define ROL(v, n) ((v) << (n) | (v) >> (sizeof(v) * CHAR_BIT - (n))) +#define HASH(h, c) ((c) + ROL(h ,7)) + struct pos { int pos, len; }; @@ -31,9 +35,11 @@ /* count the lines */ i = 1; /* extra line for sentinel */ - for (p = a; p < a + len; p++) - if (*p == '\n' || p == plast) + for (p = a; p < plast; p++) + if (*p == '\n') i++; + if (p == plast) + i++; *lr = l = (struct bdiff_line *)malloc(sizeof(struct bdiff_line) * i); if (!l) @@ -41,11 +47,10 @@ /* build the line array and calculate hashes */ hash = 0; - for (p = a; p < a + len; p++) { - /* Leonid Yuriev's hash */ - hash = (hash * 1664525) + (unsigned char)*p + 1013904223; + for (p = a; p < plast; p++) { + hash = HASH(hash, *p); - if (*p == '\n' || p == plast) { + if (*p == '\n') { l->hash = hash; hash = 0; l->len = p - b + 1; @@ -56,6 +61,15 @@ } } + if (p == plast) { + hash = HASH(hash, *p); + l->hash = hash; + l->len = p - b + 1; + l->l = b; + l->n = INT_MAX; + l++; + } + /* set up a sentinel */ l->hash = 0; l->len = 0; @@ -138,7 +152,7 @@ struct pos *pos, int a1, int a2, int b1, int b2, int *omi, int *omj) { - int mi = a1, mj = b1, mk = 0, i, j, k, half; + int mi = a1, mj = b1, mk = 0, i, j, k, half, bhalf; /* window our search on large regions to better bound worst-case performance. by choosing a window at the end, we @@ -146,7 +160,8 @@ if (a2 - a1 > 30000) a1 = a2 - 30000; - half = (a1 + a2) / 2; + half = (a1 + a2 - 1) / 2; + bhalf = (b1 + b2 - 1) / 2; for (i = a1; i < a2; i++) { /* skip all lines in b after the current block */ @@ -172,10 +187,20 @@ /* best match so far? we prefer matches closer to the middle to balance recursion */ - if (k > mk || (k == mk && (i <= mi || i < half))) { + if (k > mk) { + /* a longer match */ mi = i; mj = j; mk = k; + } else if (k == mk) { + if (i > mi && i <= half && j > b1) { + /* same match but closer to half */ + mi = i; + mj = j; + } else if (i == mi && (mj > bhalf || i == a1)) { + /* same i but best earlier j */ + mj = j; + } } } }
--- a/mercurial/bookmarks.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/bookmarks.py Sun Nov 27 20:44:52 2016 -0500 @@ -284,17 +284,21 @@ lockmod.release(tr, lock) return update -def listbookmarks(repo): +def listbinbookmarks(repo): # We may try to list bookmarks on a repo type that does not # support it (e.g., statichttprepository). marks = getattr(repo, '_bookmarks', {}) - d = {} hasnode = repo.changelog.hasnode for k, v in marks.iteritems(): # don't expose local divergent bookmarks if hasnode(v) and ('@' not in k or k.endswith('@')): - d[k] = hex(v) + yield k, v + +def listbookmarks(repo): + d = {} + for book, node in listbinbookmarks(repo): + d[book] = hex(node) return d def pushbookmark(repo, key, old, new):
--- a/mercurial/bundle2.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/bundle2.py Sun Nov 27 20:44:52 2016 -0500 @@ -485,11 +485,11 @@ return '\n'.join(chunks) bundletypes = { - "": ("", None), # only when using unbundle on ssh and old http servers + "": ("", 'UN'), # only when using unbundle on ssh and old http servers # since the unification ssh accepts a header but there # is no capability signaling it. "HG20": (), # special-cased below - "HG10UN": ("HG10UN", None), + "HG10UN": ("HG10UN", 'UN'), "HG10BZ": ("HG10", 'BZ'), "HG10GZ": ("HG10GZ", 'GZ'), } @@ -511,15 +511,15 @@ self._params = [] self._parts = [] self.capabilities = dict(capabilities) - self._compressor = util.compressors[None]() + self._compengine = util.compengines.forbundletype('UN') def setcompression(self, alg): """setup core part compression to <alg>""" - if alg is None: + if alg in (None, 'UN'): return assert not any(n.lower() == 'Compression' for n, v in self._params) self.addparam('Compression', alg) - self._compressor = util.compressors[alg]() + self._compengine = util.compengines.forbundletype(alg) @property def nbparts(self): @@ -571,12 +571,8 @@ yield _pack(_fstreamparamsize, len(param)) if param: yield param - # starting compression - for chunk in self._getcorechunk(): - data = self._compressor.compress(chunk) - if data: - yield data - yield self._compressor.flush() + for chunk in self._compengine.compressstream(self._getcorechunk()): + yield chunk def _paramchunk(self): """return a encoded version of all stream parameters""" @@ -680,7 +676,7 @@ def __init__(self, ui, fp): """If header is specified, we do not read it out of the stream.""" self.ui = ui - self._decompressor = util.decompressors[None] + self._compengine = util.compengines.forbundletype('UN') self._compressed = None super(unbundle20, self).__init__(fp) @@ -754,9 +750,9 @@ params = self._readexact(paramssize) self._processallparams(params) yield params - assert self._decompressor is util.decompressors[None] + assert self._compengine.bundletype == 'UN' # From there, payload might need to be decompressed - self._fp = self._decompressor(self._fp) + self._fp = self._compengine.decompressorreader(self._fp) emptycount = 0 while emptycount < 2: # so we can brainlessly loop @@ -780,7 +776,7 @@ # make sure param have been loaded self.params # From there, payload need to be decompressed - self._fp = self._decompressor(self._fp) + self._fp = self._compengine.decompressorreader(self._fp) indebug(self.ui, 'start extraction of bundle2 parts') headerblock = self._readpartheader() while headerblock is not None: @@ -822,10 +818,10 @@ @b2streamparamhandler('compression') def processcompression(unbundler, param, value): """read compression parameter and install payload decompression""" - if value not in util.decompressors: + if value not in util.compengines.supportedbundletypes: raise error.BundleUnknownFeatureError(params=(param,), values=(value,)) - unbundler._decompressor = util.decompressors[value] + unbundler._compengine = util.compengines.forbundletype(value) if value is not None: unbundler._compressed = True @@ -1318,18 +1314,14 @@ raise error.Abort(_('old bundle types only supports v1 ' 'changegroups')) header, comp = bundletypes[bundletype] - if comp not in util.compressors: + if comp not in util.compengines.supportedbundletypes: raise error.Abort(_('unknown stream compression type: %s') % comp) - z = util.compressors[comp]() - subchunkiter = cg.getchunks() + compengine = util.compengines.forbundletype(comp) def chunkiter(): yield header - for chunk in subchunkiter: - data = z.compress(chunk) - if data: - yield data - yield z.flush() + for chunk in compengine.compressstream(cg.getchunks()): + yield chunk chunkiter = chunkiter() # parse the changegroup data, otherwise we will block
--- a/mercurial/bundlerepo.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/bundlerepo.py Sun Nov 27 20:44:52 2016 -0500 @@ -35,6 +35,7 @@ node as nodemod, pathutil, phases, + pycompat, revlog, scmutil, util, @@ -187,9 +188,9 @@ finally: self.filteredrevs = oldfilter -class bundlemanifest(bundlerevlog, manifest.manifest): +class bundlemanifest(bundlerevlog, manifest.manifestrevlog): def __init__(self, opener, bundle, linkmapper, dirlogstarts=None, dir=''): - manifest.manifest.__init__(self, opener, dir=dir) + manifest.manifestrevlog.__init__(self, opener, dir=dir) bundlerevlog.__init__(self, opener, self.indexfile, bundle, linkmapper) if dirlogstarts is None: @@ -207,7 +208,7 @@ if node in self.fulltextcache: result = self.fulltextcache[node].tostring() else: - result = manifest.manifest.revision(self, nodeorrev) + result = manifest.manifestrevlog.revision(self, nodeorrev) return result def dirlog(self, d): @@ -359,7 +360,7 @@ @localrepo.unfilteredpropertycache def filestart(self): - self.manifest + self.manifestlog return self.filestart def url(self): @@ -392,7 +393,7 @@ return bundlepeer(self) def getcwd(self): - return os.getcwd() # always outside the repo + return pycompat.getcwd() # always outside the repo # Check if parents exist in localrepo before setting def setparents(self, p1, p2=nullid): @@ -412,13 +413,13 @@ parentpath = ui.config("bundle", "mainreporoot", "") if not parentpath: # try to find the correct path to the working directory repo - parentpath = cmdutil.findrepo(os.getcwd()) + parentpath = cmdutil.findrepo(pycompat.getcwd()) if parentpath is None: parentpath = '' if parentpath: # Try to make the full path relative so we get a nice, short URL. # In particular, we don't want temp dir names in test outputs. - cwd = os.getcwd() + cwd = pycompat.getcwd() if parentpath == cwd: parentpath = '' else:
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/cffi/bdiff.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,31 @@ +from __future__ import absolute_import + +import cffi +import os + +ffi = cffi.FFI() +ffi.set_source("_bdiff_cffi", + open(os.path.join(os.path.join(os.path.dirname(__file__), '..'), + 'bdiff.c')).read(), include_dirs=['mercurial']) +ffi.cdef(""" +struct bdiff_line { + int hash, n, e; + ssize_t len; + const char *l; +}; + +struct bdiff_hunk; +struct bdiff_hunk { + int a1, a2, b1, b2; + struct bdiff_hunk *next; +}; + +int bdiff_splitlines(const char *a, ssize_t len, struct bdiff_line **lr); +int bdiff_diff(struct bdiff_line *a, int an, struct bdiff_line *b, int bn, + struct bdiff_hunk *base); +void bdiff_freehunks(struct bdiff_hunk *l); +void free(void*); +""") + +if __name__ == '__main__': + ffi.compile()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/cffi/mpatch.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,35 @@ +from __future__ import absolute_import + +import cffi +import os + +ffi = cffi.FFI() +mpatch_c = os.path.join(os.path.join(os.path.dirname(__file__), '..', + 'mpatch.c')) +ffi.set_source("_mpatch_cffi", open(mpatch_c).read(), + include_dirs=["mercurial"]) +ffi.cdef(""" + +struct mpatch_frag { + int start, end, len; + const char *data; +}; + +struct mpatch_flist { + struct mpatch_frag *base, *head, *tail; +}; + +extern "Python" struct mpatch_flist* cffi_get_next_item(void*, ssize_t); + +int mpatch_decode(const char *bin, ssize_t len, struct mpatch_flist** res); +ssize_t mpatch_calcsize(size_t len, struct mpatch_flist *l); +void mpatch_lfree(struct mpatch_flist *a); +static int mpatch_apply(char *buf, const char *orig, size_t len, + struct mpatch_flist *l); +struct mpatch_flist *mpatch_fold(void *bins, + struct mpatch_flist* (*get_next_item)(void*, ssize_t), + ssize_t start, ssize_t end); +""") + +if __name__ == '__main__': + ffi.compile()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/cffi/osutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,102 @@ +from __future__ import absolute_import + +import cffi + +ffi = cffi.FFI() +ffi.set_source("_osutil_cffi", """ +#include <sys/attr.h> +#include <sys/vnode.h> +#include <unistd.h> +#include <fcntl.h> +#include <time.h> + +typedef struct val_attrs { + uint32_t length; + attribute_set_t returned; + attrreference_t name_info; + fsobj_type_t obj_type; + struct timespec mtime; + uint32_t accessmask; + off_t datalength; +} __attribute__((aligned(4), packed)) val_attrs_t; +""", include_dirs=['mercurial']) +ffi.cdef(''' + +typedef uint32_t attrgroup_t; + +typedef struct attrlist { + uint16_t bitmapcount; /* number of attr. bit sets in list */ + uint16_t reserved; /* (to maintain 4-byte alignment) */ + attrgroup_t commonattr; /* common attribute group */ + attrgroup_t volattr; /* volume attribute group */ + attrgroup_t dirattr; /* directory attribute group */ + attrgroup_t fileattr; /* file attribute group */ + attrgroup_t forkattr; /* fork attribute group */ + ...; +}; + +typedef struct attribute_set { + ...; +} attribute_set_t; + +typedef struct attrreference { + int attr_dataoffset; + int attr_length; + ...; +} attrreference_t; + +typedef int ... off_t; + +typedef struct val_attrs { + uint32_t length; + attribute_set_t returned; + attrreference_t name_info; + uint32_t obj_type; + struct timespec mtime; + uint32_t accessmask; + off_t datalength; + ...; +} val_attrs_t; + +/* the exact layout of the above struct will be figured out during build time */ + +typedef int ... time_t; + +typedef struct timespec { + time_t tv_sec; + ...; +}; + +int getattrlist(const char* path, struct attrlist * attrList, void * attrBuf, + size_t attrBufSize, unsigned int options); + +int getattrlistbulk(int dirfd, struct attrlist * attrList, void * attrBuf, + size_t attrBufSize, uint64_t options); + +#define ATTR_BIT_MAP_COUNT ... +#define ATTR_CMN_NAME ... +#define ATTR_CMN_OBJTYPE ... +#define ATTR_CMN_MODTIME ... +#define ATTR_CMN_ACCESSMASK ... +#define ATTR_CMN_ERROR ... +#define ATTR_CMN_RETURNED_ATTRS ... +#define ATTR_FILE_DATALENGTH ... + +#define VREG ... +#define VDIR ... +#define VLNK ... +#define VBLK ... +#define VCHR ... +#define VFIFO ... +#define VSOCK ... + +#define S_IFMT ... + +int open(const char *path, int oflag, int perm); +int close(int); + +#define O_RDONLY ... +''') + +if __name__ == '__main__': + ffi.compile()
--- a/mercurial/changegroup.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/changegroup.py Sun Nov 27 20:44:52 2016 -0500 @@ -137,14 +137,16 @@ _grouplistcount = 1 # One list of files after the manifests def __init__(self, fh, alg, extras=None): - if alg == 'UN': - alg = None # get more modern without breaking too much - if not alg in util.decompressors: + if alg is None: + alg = 'UN' + if alg not in util.compengines.supportedbundletypes: raise error.Abort(_('unknown stream compression type: %s') % alg) if alg == 'BZ': alg = '_truncatedBZ' - self._stream = util.decompressors[alg](fh) + + compengine = util.compengines.forbundletype(alg) + self._stream = compengine.decompressorreader(fh) self._type = alg self.extras = extras or {} self.callback = None @@ -250,7 +252,7 @@ # no new manifest will be created and the manifest group will # be empty during the pull self.manifestheader() - repo.manifest.addgroup(self, revmap, trp) + repo.manifestlog._revlog.addgroup(self, revmap, trp) repo.ui.progress(_('manifests'), None) self.callback = None @@ -330,11 +332,12 @@ needfiles = {} if repo.ui.configbool('server', 'validate', default=False): + cl = repo.changelog + ml = repo.manifestlog # validate incoming csets have their manifests for cset in xrange(clstart, clend): - mfnode = repo.changelog.read( - repo.changelog.node(cset))[0] - mfest = repo.manifestlog[mfnode].readdelta() + mfnode = cl.changelogrevision(cset).manifest + mfest = ml[mfnode].readdelta() # store file nodes we must see for f, n in mfest.iteritems(): needfiles.setdefault(f, set()).add(n) @@ -479,7 +482,7 @@ # If we get here, there are directory manifests in the changegroup d = chunkdata["filename"] repo.ui.debug("adding %s revisions\n" % d) - dirlog = repo.manifest.dirlog(d) + dirlog = repo.manifestlog._revlog.dirlog(d) if not dirlog.addgroup(self, revmap, trp): raise error.Abort(_("received dir revlog group is empty")) @@ -587,7 +590,7 @@ def _packmanifests(self, dir, mfnodes, lookuplinknode): """Pack flat manifests into a changegroup stream.""" assert not dir - for chunk in self.group(mfnodes, self._repo.manifest, + for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, lookuplinknode, units=_('manifests')): yield chunk @@ -676,7 +679,8 @@ def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes): repo = self._repo - dirlog = repo.manifest.dirlog + mfl = repo.manifestlog + dirlog = mfl._revlog.dirlog tmfnodes = {'': mfs} # Callback for the manifest, used to collect linkrevs for filelog @@ -704,7 +708,7 @@ treemanifests to send. """ clnode = tmfnodes[dir][x] - mdata = dirlog(dir).readshallowfast(x) + mdata = mfl.get(dir, x).readfast(shallow=True) for p, n, fl in mdata.iterentries(): if fl == 't': # subdirectory manifest subdir = dir + p + '/' @@ -850,8 +854,10 @@ def _packmanifests(self, dir, mfnodes, lookuplinknode): if dir: yield self.fileheader(dir) - for chunk in self.group(mfnodes, self._repo.manifest.dirlog(dir), - lookuplinknode, units=_('manifests')): + + dirlog = self._repo.manifestlog._revlog.dirlog(dir) + for chunk in self.group(mfnodes, dirlog, lookuplinknode, + units=_('manifests')): yield chunk def _manifestsdone(self):
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/chgserver.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,636 @@ +# chgserver.py - command server extension for cHg +# +# Copyright 2011 Yuya Nishihara <yuya@tcha.org> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +"""command server extension for cHg + +'S' channel (read/write) + propagate ui.system() request to client + +'attachio' command + attach client's stdio passed by sendmsg() + +'chdir' command + change current directory + +'getpager' command + checks if pager is enabled and which pager should be executed + +'setenv' command + replace os.environ completely + +'setumask' command + set umask + +'validate' command + reload the config and check if the server is up to date + +Config +------ + +:: + + [chgserver] + idletimeout = 3600 # seconds, after which an idle server will exit + skiphash = False # whether to skip config or env change checks +""" + +from __future__ import absolute_import + +import errno +import hashlib +import inspect +import os +import re +import signal +import struct +import sys +import time + +from .i18n import _ + +from . import ( + cmdutil, + commandserver, + error, + extensions, + osutil, + util, +) + +_log = commandserver.log + +def _hashlist(items): + """return sha1 hexdigest for a list""" + return hashlib.sha1(str(items)).hexdigest() + +# sensitive config sections affecting confighash +_configsections = [ + 'alias', # affects global state commands.table + 'extdiff', # uisetup will register new commands + 'extensions', +] + +# sensitive environment variables affecting confighash +_envre = re.compile(r'''\A(?: + CHGHG + |HG.* + |LANG(?:UAGE)? + |LC_.* + |LD_.* + |PATH + |PYTHON.* + |TERM(?:INFO)? + |TZ + )\Z''', re.X) + +def _confighash(ui): + """return a quick hash for detecting config/env changes + + confighash is the hash of sensitive config items and environment variables. + + for chgserver, it is designed that once confighash changes, the server is + not qualified to serve its client and should redirect the client to a new + server. different from mtimehash, confighash change will not mark the + server outdated and exit since the user can have different configs at the + same time. + """ + sectionitems = [] + for section in _configsections: + sectionitems.append(ui.configitems(section)) + sectionhash = _hashlist(sectionitems) + envitems = [(k, v) for k, v in os.environ.iteritems() if _envre.match(k)] + envhash = _hashlist(sorted(envitems)) + return sectionhash[:6] + envhash[:6] + +def _getmtimepaths(ui): + """get a list of paths that should be checked to detect change + + The list will include: + - extensions (will not cover all files for complex extensions) + - mercurial/__version__.py + - python binary + """ + modules = [m for n, m in extensions.extensions(ui)] + try: + from . import __version__ + modules.append(__version__) + except ImportError: + pass + files = [sys.executable] + for m in modules: + try: + files.append(inspect.getabsfile(m)) + except TypeError: + pass + return sorted(set(files)) + +def _mtimehash(paths): + """return a quick hash for detecting file changes + + mtimehash calls stat on given paths and calculate a hash based on size and + mtime of each file. mtimehash does not read file content because reading is + expensive. therefore it's not 100% reliable for detecting content changes. + it's possible to return different hashes for same file contents. + it's also possible to return a same hash for different file contents for + some carefully crafted situation. + + for chgserver, it is designed that once mtimehash changes, the server is + considered outdated immediately and should no longer provide service. + + mtimehash is not included in confighash because we only know the paths of + extensions after importing them (there is imp.find_module but that faces + race conditions). We need to calculate confighash without importing. + """ + def trystat(path): + try: + st = os.stat(path) + return (st.st_mtime, st.st_size) + except OSError: + # could be ENOENT, EPERM etc. not fatal in any case + pass + return _hashlist(map(trystat, paths))[:12] + +class hashstate(object): + """a structure storing confighash, mtimehash, paths used for mtimehash""" + def __init__(self, confighash, mtimehash, mtimepaths): + self.confighash = confighash + self.mtimehash = mtimehash + self.mtimepaths = mtimepaths + + @staticmethod + def fromui(ui, mtimepaths=None): + if mtimepaths is None: + mtimepaths = _getmtimepaths(ui) + confighash = _confighash(ui) + mtimehash = _mtimehash(mtimepaths) + _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash)) + return hashstate(confighash, mtimehash, mtimepaths) + +# copied from hgext/pager.py:uisetup() +def _setuppagercmd(ui, options, cmd): + from . import commands # avoid cycle + + if not ui.formatted(): + return + + p = ui.config("pager", "pager", os.environ.get("PAGER")) + usepager = False + always = util.parsebool(options['pager']) + auto = options['pager'] == 'auto' + + if not p: + pass + elif always: + usepager = True + elif not auto: + usepager = False + else: + attended = ['annotate', 'cat', 'diff', 'export', 'glog', 'log', 'qdiff'] + attend = ui.configlist('pager', 'attend', attended) + ignore = ui.configlist('pager', 'ignore') + cmds, _ = cmdutil.findcmd(cmd, commands.table) + + for cmd in cmds: + var = 'attend-%s' % cmd + if ui.config('pager', var): + usepager = ui.configbool('pager', var) + break + if (cmd in attend or + (cmd not in ignore and not attend)): + usepager = True + break + + if usepager: + ui.setconfig('ui', 'formatted', ui.formatted(), 'pager') + ui.setconfig('ui', 'interactive', False, 'pager') + return p + +def _newchgui(srcui, csystem): + class chgui(srcui.__class__): + def __init__(self, src=None): + super(chgui, self).__init__(src) + if src: + self._csystem = getattr(src, '_csystem', csystem) + else: + self._csystem = csystem + + def system(self, cmd, environ=None, cwd=None, onerr=None, + errprefix=None): + # fallback to the original system method if the output needs to be + # captured (to self._buffers), or the output stream is not stdout + # (e.g. stderr, cStringIO), because the chg client is not aware of + # these situations and will behave differently (write to stdout). + if (any(s[1] for s in self._bufferstates) + or not util.safehasattr(self.fout, 'fileno') + or self.fout.fileno() != util.stdout.fileno()): + return super(chgui, self).system(cmd, environ, cwd, onerr, + errprefix) + # copied from mercurial/util.py:system() + self.flush() + def py2shell(val): + if val is None or val is False: + return '0' + if val is True: + return '1' + return str(val) + env = os.environ.copy() + if environ: + env.update((k, py2shell(v)) for k, v in environ.iteritems()) + env['HG'] = util.hgexecutable() + rc = self._csystem(cmd, env, cwd) + if rc and onerr: + errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]), + util.explainexit(rc)[0]) + if errprefix: + errmsg = '%s: %s' % (errprefix, errmsg) + raise onerr(errmsg) + return rc + + return chgui(srcui) + +def _loadnewui(srcui, args): + from . import dispatch # avoid cycle + + newui = srcui.__class__() + for a in ['fin', 'fout', 'ferr', 'environ']: + setattr(newui, a, getattr(srcui, a)) + if util.safehasattr(srcui, '_csystem'): + newui._csystem = srcui._csystem + + # command line args + args = args[:] + dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args)) + + # stolen from tortoisehg.util.copydynamicconfig() + for section, name, value in srcui.walkconfig(): + source = srcui.configsource(section, name) + if ':' in source or source == '--config': + # path:line or command line + continue + if source == 'none': + # ui.configsource returns 'none' by default + source = '' + newui.setconfig(section, name, value, source) + + # load wd and repo config, copied from dispatch.py + cwds = dispatch._earlygetopt(['--cwd'], args) + cwd = cwds and os.path.realpath(cwds[-1]) or None + rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args) + path, newlui = dispatch._getlocal(newui, rpath, wd=cwd) + + return (newui, newlui) + +class channeledsystem(object): + """Propagate ui.system() request in the following format: + + payload length (unsigned int), + cmd, '\0', + cwd, '\0', + envkey, '=', val, '\0', + ... + envkey, '=', val + + and waits: + + exitcode length (unsigned int), + exitcode (int) + """ + def __init__(self, in_, out, channel): + self.in_ = in_ + self.out = out + self.channel = channel + + def __call__(self, cmd, environ, cwd): + args = [util.quotecommand(cmd), os.path.abspath(cwd or '.')] + args.extend('%s=%s' % (k, v) for k, v in environ.iteritems()) + data = '\0'.join(args) + self.out.write(struct.pack('>cI', self.channel, len(data))) + self.out.write(data) + self.out.flush() + + length = self.in_.read(4) + length, = struct.unpack('>I', length) + if length != 4: + raise error.Abort(_('invalid response')) + rc, = struct.unpack('>i', self.in_.read(4)) + return rc + +_iochannels = [ + # server.ch, ui.fp, mode + ('cin', 'fin', 'rb'), + ('cout', 'fout', 'wb'), + ('cerr', 'ferr', 'wb'), +] + +class chgcmdserver(commandserver.server): + def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress): + super(chgcmdserver, self).__init__( + _newchgui(ui, channeledsystem(fin, fout, 'S')), repo, fin, fout) + self.clientsock = sock + self._oldios = [] # original (self.ch, ui.fp, fd) before "attachio" + self.hashstate = hashstate + self.baseaddress = baseaddress + if hashstate is not None: + self.capabilities = self.capabilities.copy() + self.capabilities['validate'] = chgcmdserver.validate + + def cleanup(self): + super(chgcmdserver, self).cleanup() + # dispatch._runcatch() does not flush outputs if exception is not + # handled by dispatch._dispatch() + self.ui.flush() + self._restoreio() + + def attachio(self): + """Attach to client's stdio passed via unix domain socket; all + channels except cresult will no longer be used + """ + # tell client to sendmsg() with 1-byte payload, which makes it + # distinctive from "attachio\n" command consumed by client.read() + self.clientsock.sendall(struct.pack('>cI', 'I', 1)) + clientfds = osutil.recvfds(self.clientsock.fileno()) + _log('received fds: %r\n' % clientfds) + + ui = self.ui + ui.flush() + first = self._saveio() + for fd, (cn, fn, mode) in zip(clientfds, _iochannels): + assert fd > 0 + fp = getattr(ui, fn) + os.dup2(fd, fp.fileno()) + os.close(fd) + if not first: + continue + # reset buffering mode when client is first attached. as we want + # to see output immediately on pager, the mode stays unchanged + # when client re-attached. ferr is unchanged because it should + # be unbuffered no matter if it is a tty or not. + if fn == 'ferr': + newfp = fp + else: + # make it line buffered explicitly because the default is + # decided on first write(), where fout could be a pager. + if fp.isatty(): + bufsize = 1 # line buffered + else: + bufsize = -1 # system default + newfp = os.fdopen(fp.fileno(), mode, bufsize) + setattr(ui, fn, newfp) + setattr(self, cn, newfp) + + self.cresult.write(struct.pack('>i', len(clientfds))) + + def _saveio(self): + if self._oldios: + return False + ui = self.ui + for cn, fn, _mode in _iochannels: + ch = getattr(self, cn) + fp = getattr(ui, fn) + fd = os.dup(fp.fileno()) + self._oldios.append((ch, fp, fd)) + return True + + def _restoreio(self): + ui = self.ui + for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels): + newfp = getattr(ui, fn) + # close newfp while it's associated with client; otherwise it + # would be closed when newfp is deleted + if newfp is not fp: + newfp.close() + # restore original fd: fp is open again + os.dup2(fd, fp.fileno()) + os.close(fd) + setattr(self, cn, ch) + setattr(ui, fn, fp) + del self._oldios[:] + + def validate(self): + """Reload the config and check if the server is up to date + + Read a list of '\0' separated arguments. + Write a non-empty list of '\0' separated instruction strings or '\0' + if the list is empty. + An instruction string could be either: + - "unlink $path", the client should unlink the path to stop the + outdated server. + - "redirect $path", the client should attempt to connect to $path + first. If it does not work, start a new server. It implies + "reconnect". + - "exit $n", the client should exit directly with code n. + This may happen if we cannot parse the config. + - "reconnect", the client should close the connection and + reconnect. + If neither "reconnect" nor "redirect" is included in the instruction + list, the client can continue with this server after completing all + the instructions. + """ + from . import dispatch # avoid cycle + + args = self._readlist() + try: + self.ui, lui = _loadnewui(self.ui, args) + except error.ParseError as inst: + dispatch._formatparse(self.ui.warn, inst) + self.ui.flush() + self.cresult.write('exit 255') + return + newhash = hashstate.fromui(lui, self.hashstate.mtimepaths) + insts = [] + if newhash.mtimehash != self.hashstate.mtimehash: + addr = _hashaddress(self.baseaddress, self.hashstate.confighash) + insts.append('unlink %s' % addr) + # mtimehash is empty if one or more extensions fail to load. + # to be compatible with hg, still serve the client this time. + if self.hashstate.mtimehash: + insts.append('reconnect') + if newhash.confighash != self.hashstate.confighash: + addr = _hashaddress(self.baseaddress, newhash.confighash) + insts.append('redirect %s' % addr) + _log('validate: %s\n' % insts) + self.cresult.write('\0'.join(insts) or '\0') + + def chdir(self): + """Change current directory + + Note that the behavior of --cwd option is bit different from this. + It does not affect --config parameter. + """ + path = self._readstr() + if not path: + return + _log('chdir to %r\n' % path) + os.chdir(path) + + def setumask(self): + """Change umask""" + mask = struct.unpack('>I', self._read(4))[0] + _log('setumask %r\n' % mask) + os.umask(mask) + + def getpager(self): + """Read cmdargs and write pager command to r-channel if enabled + + If pager isn't enabled, this writes '\0' because channeledoutput + does not allow to write empty data. + """ + from . import dispatch # avoid cycle + + args = self._readlist() + try: + cmd, _func, args, options, _cmdoptions = dispatch._parse(self.ui, + args) + except (error.Abort, error.AmbiguousCommand, error.CommandError, + error.UnknownCommand): + cmd = None + options = {} + if not cmd or 'pager' not in options: + self.cresult.write('\0') + return + + pagercmd = _setuppagercmd(self.ui, options, cmd) + if pagercmd: + # Python's SIGPIPE is SIG_IGN by default. change to SIG_DFL so + # we can exit if the pipe to the pager is closed + if util.safehasattr(signal, 'SIGPIPE') and \ + signal.getsignal(signal.SIGPIPE) == signal.SIG_IGN: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + self.cresult.write(pagercmd) + else: + self.cresult.write('\0') + + def setenv(self): + """Clear and update os.environ + + Note that not all variables can make an effect on the running process. + """ + l = self._readlist() + try: + newenv = dict(s.split('=', 1) for s in l) + except ValueError: + raise ValueError('unexpected value in setenv request') + _log('setenv: %r\n' % sorted(newenv.keys())) + os.environ.clear() + os.environ.update(newenv) + + capabilities = commandserver.server.capabilities.copy() + capabilities.update({'attachio': attachio, + 'chdir': chdir, + 'getpager': getpager, + 'setenv': setenv, + 'setumask': setumask}) + +def _tempaddress(address): + return '%s.%d.tmp' % (address, os.getpid()) + +def _hashaddress(address, hashstr): + return '%s-%s' % (address, hashstr) + +class chgunixservicehandler(object): + """Set of operations for chg services""" + + pollinterval = 1 # [sec] + + def __init__(self, ui): + self.ui = ui + self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600) + self._lastactive = time.time() + + def bindsocket(self, sock, address): + self._inithashstate(address) + self._checkextensions() + self._bind(sock) + self._createsymlink() + + def _inithashstate(self, address): + self._baseaddress = address + if self.ui.configbool('chgserver', 'skiphash', False): + self._hashstate = None + self._realaddress = address + return + self._hashstate = hashstate.fromui(self.ui) + self._realaddress = _hashaddress(address, self._hashstate.confighash) + + def _checkextensions(self): + if not self._hashstate: + return + if extensions.notloaded(): + # one or more extensions failed to load. mtimehash becomes + # meaningless because we do not know the paths of those extensions. + # set mtimehash to an illegal hash value to invalidate the server. + self._hashstate.mtimehash = '' + + def _bind(self, sock): + # use a unique temp address so we can stat the file and do ownership + # check later + tempaddress = _tempaddress(self._realaddress) + util.bindunixsocket(sock, tempaddress) + self._socketstat = os.stat(tempaddress) + # rename will replace the old socket file if exists atomically. the + # old server will detect ownership change and exit. + util.rename(tempaddress, self._realaddress) + + def _createsymlink(self): + if self._baseaddress == self._realaddress: + return + tempaddress = _tempaddress(self._baseaddress) + os.symlink(os.path.basename(self._realaddress), tempaddress) + util.rename(tempaddress, self._baseaddress) + + def _issocketowner(self): + try: + stat = os.stat(self._realaddress) + return (stat.st_ino == self._socketstat.st_ino and + stat.st_mtime == self._socketstat.st_mtime) + except OSError: + return False + + def unlinksocket(self, address): + if not self._issocketowner(): + return + # it is possible to have a race condition here that we may + # remove another server's socket file. but that's okay + # since that server will detect and exit automatically and + # the client will start a new server on demand. + try: + os.unlink(self._realaddress) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + def printbanner(self, address): + # no "listening at" message should be printed to simulate hg behavior + pass + + def shouldexit(self): + if not self._issocketowner(): + self.ui.debug('%s is not owned, exiting.\n' % self._realaddress) + return True + if time.time() - self._lastactive > self._idletimeout: + self.ui.debug('being idle too long. exiting.\n') + return True + return False + + def newconnection(self): + self._lastactive = time.time() + + def createcmdserver(self, repo, conn, fin, fout): + return chgcmdserver(self.ui, repo, fin, fout, conn, + self._hashstate, self._baseaddress) + +def chgunixservice(ui, repo, opts): + # CHGINTERNALMARK is temporarily set by chg client to detect if chg will + # start another chg. drop it to avoid possible side effects. + if 'CHGINTERNALMARK' in os.environ: + del os.environ['CHGINTERNALMARK'] + + if repo: + # one chgserver can serve multiple repos. drop repo information + ui.setconfig('bundle', 'mainreporoot', '', 'repo') + h = chgunixservicehandler(ui) + return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h)
--- a/mercurial/cmdutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/cmdutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -10,7 +10,6 @@ import errno import os import re -import sys import tempfile from .i18n import _ @@ -27,16 +26,19 @@ changelog, copies, crecord as crecordmod, + dirstateguard as dirstateguardmod, encoding, error, formatter, graphmod, lock as lockmod, match as matchmod, + mergeutil, obsolete, patch, pathutil, phases, + pycompat, repair, revlog, revset, @@ -83,7 +85,7 @@ else: recordfn = crecordmod.chunkselector - return crecordmod.filterpatch(ui, originalhunks, recordfn) + return crecordmod.filterpatch(ui, originalhunks, recordfn, operation) else: return patch.filterpatch(ui, originalhunks, operation) @@ -555,11 +557,11 @@ if 'treemanifest' not in repo.requirements: raise error.Abort(_("--dir can only be used on repos with " "treemanifest enabled")) - dirlog = repo.manifest.dirlog(dir) + dirlog = repo.manifestlog._revlog.dirlog(dir) if len(dirlog): r = dirlog elif mf: - r = repo.manifest + r = repo.manifestlog._revlog elif file_: filelog = repo.file(file_) if len(filelog): @@ -569,7 +571,7 @@ raise error.CommandError(cmd, _('invalid arguments')) if not os.path.isfile(file_): raise error.Abort(_("revlog '%s' not found") % file_) - r = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), + r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), file_[:-2] + ".i") return r @@ -819,93 +821,6 @@ return errors != 0 -def service(opts, parentfn=None, initfn=None, runfn=None, logfile=None, - runargs=None, appendpid=False): - '''Run a command as a service.''' - - def writepid(pid): - if opts['pid_file']: - if appendpid: - mode = 'a' - else: - mode = 'w' - fp = open(opts['pid_file'], mode) - fp.write(str(pid) + '\n') - fp.close() - - if opts['daemon'] and not opts['daemon_postexec']: - # Signal child process startup with file removal - lockfd, lockpath = tempfile.mkstemp(prefix='hg-service-') - os.close(lockfd) - try: - if not runargs: - runargs = util.hgcmd() + sys.argv[1:] - runargs.append('--daemon-postexec=unlink:%s' % lockpath) - # Don't pass --cwd to the child process, because we've already - # changed directory. - for i in xrange(1, len(runargs)): - if runargs[i].startswith('--cwd='): - del runargs[i] - break - elif runargs[i].startswith('--cwd'): - del runargs[i:i + 2] - break - def condfn(): - return not os.path.exists(lockpath) - pid = util.rundetached(runargs, condfn) - if pid < 0: - raise error.Abort(_('child process failed to start')) - writepid(pid) - finally: - try: - os.unlink(lockpath) - except OSError as e: - if e.errno != errno.ENOENT: - raise - if parentfn: - return parentfn(pid) - else: - return - - if initfn: - initfn() - - if not opts['daemon']: - writepid(util.getpid()) - - if opts['daemon_postexec']: - try: - os.setsid() - except AttributeError: - pass - for inst in opts['daemon_postexec']: - if inst.startswith('unlink:'): - lockpath = inst[7:] - os.unlink(lockpath) - elif inst.startswith('chdir:'): - os.chdir(inst[6:]) - elif inst != 'none': - raise error.Abort(_('invalid value for --daemon-postexec: %s') - % inst) - util.hidewindow() - sys.stdout.flush() - sys.stderr.flush() - - nullfd = os.open(os.devnull, os.O_RDWR) - logfilefd = nullfd - if logfile: - logfilefd = os.open(logfile, os.O_RDWR | os.O_CREAT | os.O_APPEND) - os.dup2(nullfd, 0) - os.dup2(logfilefd, 1) - os.dup2(logfilefd, 2) - if nullfd not in (0, 1, 2): - os.close(nullfd) - if logfile and logfilefd not in (0, 1, 2): - os.close(logfilefd) - - if runfn: - return runfn() - ## facility to let extension process additional data into an import patch # list of identifier to be executed in order extrapreimport = [] # run before commit @@ -1202,8 +1117,7 @@ chunks = patch.diff(repo, node1, node2, match, changes, diffopts, prefix=prefix, relroot=relroot) for chunk, label in patch.diffstatui(util.iterlines(chunks), - width=width, - git=diffopts.git): + width=width): write(chunk, label=label) else: for chunk, label in patch.diffui(repo, node1, node2, match, @@ -1324,7 +1238,8 @@ mnode = ctx.manifestnode() # i18n: column positioning for "hg log" self.ui.write(_("manifest: %d:%s\n") % - (self.repo.manifest.rev(mnode), hex(mnode)), + (self.repo.manifestlog._revlog.rev(mnode), + hex(mnode)), label='ui.debug log.manifest') # i18n: column positioning for "hg log" self.ui.write(_("user: %s\n") % ctx.user(), @@ -2566,11 +2481,14 @@ # for performance to avoid the cost of parsing the manifest. if len(matcher.files()) == 1 and not matcher.anypats(): file = matcher.files()[0] - mf = repo.manifest + mfl = repo.manifestlog mfnode = ctx.manifestnode() - if mfnode and mf.find(mfnode, file)[0]: - write(file) - return 0 + try: + if mfnode and mfl[mfnode].find(file)[0]: + write(file) + return 0 + except KeyError: + pass for abs in ctx.walk(matcher): write(abs) @@ -2827,7 +2745,7 @@ committext = buildcommittext(repo, ctx, subs, extramsg) # run editor in the repository root - olddir = os.getcwd() + olddir = pycompat.getcwd() os.chdir(repo.root) # make in-memory changes visible to external process @@ -3249,13 +3167,18 @@ fc = ctx[f] repo.wwrite(f, fc.data(), fc.flags()) + def doremove(f): + try: + util.unlinkpath(repo.wjoin(f)) + except OSError: + pass + repo.dirstate.remove(f) + audit_path = pathutil.pathauditor(repo.root) for f in actions['forget'][0]: if interactive: - choice = \ - repo.ui.promptchoice( - _("forget added file %s (yn)?$$ &Yes $$ &No") - % f) + choice = repo.ui.promptchoice( + _("forget added file %s (Yn)?$$ &Yes $$ &No") % f) if choice == 0: repo.dirstate.drop(f) else: @@ -3264,11 +3187,15 @@ repo.dirstate.drop(f) for f in actions['remove'][0]: audit_path(f) - try: - util.unlinkpath(repo.wjoin(f)) - except OSError: - pass - repo.dirstate.remove(f) + if interactive: + choice = repo.ui.promptchoice( + _("remove added file %s (Yn)?$$ &Yes $$ &No") % f) + if choice == 0: + doremove(f) + else: + excluded_files.append(repo.wjoin(f)) + else: + doremove(f) for f in actions['drop'][0]: audit_path(f) repo.dirstate.remove(f) @@ -3403,6 +3330,11 @@ return cmd +def checkunresolved(ms): + ms._repo.ui.deprecwarn('checkunresolved moved from cmdutil to mergeutil', + '4.1') + return mergeutil.checkunresolved(ms) + # a list of (ui, repo, otherpeer, opts, missing) functions called by # commands.outgoing. "missing" is "missing" of the result of # "findcommonoutgoing()" @@ -3463,7 +3395,7 @@ '''Check for an unfinished operation and return the command to finish it. - afterresolvedstates tupples define a .hg/{file} and the corresponding + afterresolvedstates tuples define a .hg/{file} and the corresponding command needed to finish it. Returns a (msg, warning) tuple. 'msg' is a string and 'warning' is @@ -3510,57 +3442,9 @@ hint = after[0] raise error.Abort(_('no %s in progress') % task, hint=hint) -class dirstateguard(object): - '''Restore dirstate at unexpected failure. - - At the construction, this class does: - - - write current ``repo.dirstate`` out, and - - save ``.hg/dirstate`` into the backup file - - This restores ``.hg/dirstate`` from backup file, if ``release()`` - is invoked before ``close()``. - - This just removes the backup file at ``close()`` before ``release()``. - ''' - +class dirstateguard(dirstateguardmod.dirstateguard): def __init__(self, repo, name): - self._repo = repo - self._active = False - self._closed = False - self._suffix = '.backup.%s.%d' % (name, id(self)) - repo.dirstate.savebackup(repo.currenttransaction(), self._suffix) - self._active = True - - def __del__(self): - if self._active: # still active - # this may occur, even if this class is used correctly: - # for example, releasing other resources like transaction - # may raise exception before ``dirstateguard.release`` in - # ``release(tr, ....)``. - self._abort() - - def close(self): - if not self._active: # already inactivated - msg = (_("can't close already inactivated backup: dirstate%s") - % self._suffix) - raise error.Abort(msg) - - self._repo.dirstate.clearbackup(self._repo.currenttransaction(), - self._suffix) - self._active = False - self._closed = True - - def _abort(self): - self._repo.dirstate.restorebackup(self._repo.currenttransaction(), - self._suffix) - self._active = False - - def release(self): - if not self._closed: - if not self._active: # already inactivated - msg = (_("can't release already inactivated backup:" - " dirstate%s") - % self._suffix) - raise error.Abort(msg) - self._abort() + dirstateguardmod.dirstateguard.__init__(self, repo, name) + repo.ui.deprecwarn( + 'dirstateguard has moved from cmdutil to dirstateguard', + '4.1')
--- a/mercurial/commands.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/commands.py Sun Nov 27 20:44:52 2016 -0500 @@ -9,12 +9,11 @@ import difflib import errno -import operator import os -import random import re import shlex import socket +import string import sys import tempfile import time @@ -34,25 +33,19 @@ bundle2, changegroup, cmdutil, - commandserver, - context, copies, - dagparser, - dagutil, destutil, + dirstateguard, discovery, encoding, error, exchange, extensions, - fileset, formatter, graphmod, hbisect, help, hg, - hgweb, - localrepo, lock as lockmod, merge as mergemod, minirst, @@ -61,18 +54,17 @@ phases, policy, pvec, + pycompat, repair, revlog, revset, scmutil, - setdiscovery, - simplemerge, + server, sshserver, sslutil, streamclone, templatekw, templater, - treediscovery, ui as uimod, util, ) @@ -686,7 +678,7 @@ bheads = repo.branchheads(branch) rctx = scmutil.revsingle(repo, hex(parent)) if not opts.get('merge') and op1 != node: - dsguard = cmdutil.dirstateguard(repo, 'backout') + dsguard = dirstateguard.dirstateguard(repo, 'backout') try: ui.setconfig('ui', 'forcemerge', opts.get('tool', ''), 'backout') @@ -1866,806 +1858,6 @@ with repo.wlock(False): return cmdutil.copy(ui, repo, pats, opts) -@command('debugancestor', [], _('[INDEX] REV1 REV2'), optionalrepo=True) -def debugancestor(ui, repo, *args): - """find the ancestor revision of two revisions in a given index""" - if len(args) == 3: - index, rev1, rev2 = args - r = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), index) - lookup = r.lookup - elif len(args) == 2: - if not repo: - raise error.Abort(_("there is no Mercurial repository here " - "(.hg not found)")) - rev1, rev2 = args - r = repo.changelog - lookup = repo.lookup - else: - raise error.Abort(_('either two or three arguments required')) - a = r.ancestor(lookup(rev1), lookup(rev2)) - ui.write("%d:%s\n" % (r.rev(a), hex(a))) - -@command('debugbuilddag', - [('m', 'mergeable-file', None, _('add single file mergeable changes')), - ('o', 'overwritten-file', None, _('add single file all revs overwrite')), - ('n', 'new-file', None, _('add new file at each rev'))], - _('[OPTION]... [TEXT]')) -def debugbuilddag(ui, repo, text=None, - mergeable_file=False, - overwritten_file=False, - new_file=False): - """builds a repo with a given DAG from scratch in the current empty repo - - The description of the DAG is read from stdin if not given on the - command line. - - Elements: - - - "+n" is a linear run of n nodes based on the current default parent - - "." is a single node based on the current default parent - - "$" resets the default parent to null (implied at the start); - otherwise the default parent is always the last node created - - "<p" sets the default parent to the backref p - - "*p" is a fork at parent p, which is a backref - - "*p1/p2" is a merge of parents p1 and p2, which are backrefs - - "/p2" is a merge of the preceding node and p2 - - ":tag" defines a local tag for the preceding node - - "@branch" sets the named branch for subsequent nodes - - "#...\\n" is a comment up to the end of the line - - Whitespace between the above elements is ignored. - - A backref is either - - - a number n, which references the node curr-n, where curr is the current - node, or - - the name of a local tag you placed earlier using ":tag", or - - empty to denote the default parent. - - All string valued-elements are either strictly alphanumeric, or must - be enclosed in double quotes ("..."), with "\\" as escape character. - """ - - if text is None: - ui.status(_("reading DAG from stdin\n")) - text = ui.fin.read() - - cl = repo.changelog - if len(cl) > 0: - raise error.Abort(_('repository is not empty')) - - # determine number of revs in DAG - total = 0 - for type, data in dagparser.parsedag(text): - if type == 'n': - total += 1 - - if mergeable_file: - linesperrev = 2 - # make a file with k lines per rev - initialmergedlines = [str(i) for i in xrange(0, total * linesperrev)] - initialmergedlines.append("") - - tags = [] - - wlock = lock = tr = None - try: - wlock = repo.wlock() - lock = repo.lock() - tr = repo.transaction("builddag") - - at = -1 - atbranch = 'default' - nodeids = [] - id = 0 - ui.progress(_('building'), id, unit=_('revisions'), total=total) - for type, data in dagparser.parsedag(text): - if type == 'n': - ui.note(('node %s\n' % str(data))) - id, ps = data - - files = [] - fctxs = {} - - p2 = None - if mergeable_file: - fn = "mf" - p1 = repo[ps[0]] - if len(ps) > 1: - p2 = repo[ps[1]] - pa = p1.ancestor(p2) - base, local, other = [x[fn].data() for x in (pa, p1, - p2)] - m3 = simplemerge.Merge3Text(base, local, other) - ml = [l.strip() for l in m3.merge_lines()] - ml.append("") - elif at > 0: - ml = p1[fn].data().split("\n") - else: - ml = initialmergedlines - ml[id * linesperrev] += " r%i" % id - mergedtext = "\n".join(ml) - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, mergedtext) - - if overwritten_file: - fn = "of" - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) - - if new_file: - fn = "nf%i" % id - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) - if len(ps) > 1: - if not p2: - p2 = repo[ps[1]] - for fn in p2: - if fn.startswith("nf"): - files.append(fn) - fctxs[fn] = p2[fn] - - def fctxfn(repo, cx, path): - return fctxs.get(path) - - if len(ps) == 0 or ps[0] < 0: - pars = [None, None] - elif len(ps) == 1: - pars = [nodeids[ps[0]], None] - else: - pars = [nodeids[p] for p in ps] - cx = context.memctx(repo, pars, "r%i" % id, files, fctxfn, - date=(id, 0), - user="debugbuilddag", - extra={'branch': atbranch}) - nodeid = repo.commitctx(cx) - nodeids.append(nodeid) - at = id - elif type == 'l': - id, name = data - ui.note(('tag %s\n' % name)) - tags.append("%s %s\n" % (hex(repo.changelog.node(id)), name)) - elif type == 'a': - ui.note(('branch %s\n' % data)) - atbranch = data - ui.progress(_('building'), id, unit=_('revisions'), total=total) - tr.close() - - if tags: - repo.vfs.write("localtags", "".join(tags)) - finally: - ui.progress(_('building'), None) - release(tr, lock, wlock) - -@command('debugbundle', - [('a', 'all', None, _('show all details')), - ('', 'spec', None, _('print the bundlespec of the bundle'))], - _('FILE'), - norepo=True) -def debugbundle(ui, bundlepath, all=None, spec=None, **opts): - """lists the contents of a bundle""" - with hg.openpath(ui, bundlepath) as f: - if spec: - spec = exchange.getbundlespec(ui, f) - ui.write('%s\n' % spec) - return - - gen = exchange.readbundle(ui, f, bundlepath) - if isinstance(gen, bundle2.unbundle20): - return _debugbundle2(ui, gen, all=all, **opts) - _debugchangegroup(ui, gen, all=all, **opts) - -def _debugchangegroup(ui, gen, all=None, indent=0, **opts): - indent_string = ' ' * indent - if all: - ui.write(("%sformat: id, p1, p2, cset, delta base, len(delta)\n") - % indent_string) - - def showchunks(named): - ui.write("\n%s%s\n" % (indent_string, named)) - chain = None - for chunkdata in iter(lambda: gen.deltachunk(chain), {}): - node = chunkdata['node'] - p1 = chunkdata['p1'] - p2 = chunkdata['p2'] - cs = chunkdata['cs'] - deltabase = chunkdata['deltabase'] - delta = chunkdata['delta'] - ui.write("%s%s %s %s %s %s %s\n" % - (indent_string, hex(node), hex(p1), hex(p2), - hex(cs), hex(deltabase), len(delta))) - chain = node - - chunkdata = gen.changelogheader() - showchunks("changelog") - chunkdata = gen.manifestheader() - showchunks("manifest") - for chunkdata in iter(gen.filelogheader, {}): - fname = chunkdata['filename'] - showchunks(fname) - else: - if isinstance(gen, bundle2.unbundle20): - raise error.Abort(_('use debugbundle2 for this file')) - chunkdata = gen.changelogheader() - chain = None - for chunkdata in iter(lambda: gen.deltachunk(chain), {}): - node = chunkdata['node'] - ui.write("%s%s\n" % (indent_string, hex(node))) - chain = node - -def _debugbundle2(ui, gen, all=None, **opts): - """lists the contents of a bundle2""" - if not isinstance(gen, bundle2.unbundle20): - raise error.Abort(_('not a bundle2 file')) - ui.write(('Stream params: %s\n' % repr(gen.params))) - for part in gen.iterparts(): - ui.write('%s -- %r\n' % (part.type, repr(part.params))) - if part.type == 'changegroup': - version = part.params.get('version', '01') - cg = changegroup.getunbundler(version, part, 'UN') - _debugchangegroup(ui, cg, all=all, indent=4, **opts) - -@command('debugcreatestreamclonebundle', [], 'FILE') -def debugcreatestreamclonebundle(ui, repo, fname): - """create a stream clone bundle file - - Stream bundles are special bundles that are essentially archives of - revlog files. They are commonly used for cloning very quickly. - """ - requirements, gen = streamclone.generatebundlev1(repo) - changegroup.writechunks(ui, gen, fname) - - ui.write(_('bundle requirements: %s\n') % ', '.join(sorted(requirements))) - -@command('debugapplystreamclonebundle', [], 'FILE') -def debugapplystreamclonebundle(ui, repo, fname): - """apply a stream clone bundle file""" - f = hg.openpath(ui, fname) - gen = exchange.readbundle(ui, f, fname) - gen.apply(repo) - -@command('debugcheckstate', [], '') -def debugcheckstate(ui, repo): - """validate the correctness of the current dirstate""" - parent1, parent2 = repo.dirstate.parents() - m1 = repo[parent1].manifest() - m2 = repo[parent2].manifest() - errors = 0 - for f in repo.dirstate: - state = repo.dirstate[f] - if state in "nr" and f not in m1: - ui.warn(_("%s in state %s, but not in manifest1\n") % (f, state)) - errors += 1 - if state in "a" and f in m1: - ui.warn(_("%s in state %s, but also in manifest1\n") % (f, state)) - errors += 1 - if state in "m" and f not in m1 and f not in m2: - ui.warn(_("%s in state %s, but not in either manifest\n") % - (f, state)) - errors += 1 - for f in m1: - state = repo.dirstate[f] - if state not in "nrm": - ui.warn(_("%s in manifest1, but listed as state %s") % (f, state)) - errors += 1 - if errors: - error = _(".hg/dirstate inconsistent with current parent's manifest") - raise error.Abort(error) - -@command('debugcommands', [], _('[COMMAND]'), norepo=True) -def debugcommands(ui, cmd='', *args): - """list all available commands and options""" - for cmd, vals in sorted(table.iteritems()): - cmd = cmd.split('|')[0].strip('^') - opts = ', '.join([i[1] for i in vals[1]]) - ui.write('%s: %s\n' % (cmd, opts)) - -@command('debugcomplete', - [('o', 'options', None, _('show the command options'))], - _('[-o] CMD'), - norepo=True) -def debugcomplete(ui, cmd='', **opts): - """returns the completion list associated with the given command""" - - if opts.get('options'): - options = [] - otables = [globalopts] - if cmd: - aliases, entry = cmdutil.findcmd(cmd, table, False) - otables.append(entry[1]) - for t in otables: - for o in t: - if "(DEPRECATED)" in o[3]: - continue - if o[0]: - options.append('-%s' % o[0]) - options.append('--%s' % o[1]) - ui.write("%s\n" % "\n".join(options)) - return - - cmdlist, unused_allcmds = cmdutil.findpossible(cmd, table) - if ui.verbose: - cmdlist = [' '.join(c[0]) for c in cmdlist.values()] - ui.write("%s\n" % "\n".join(sorted(cmdlist))) - -@command('debugdag', - [('t', 'tags', None, _('use tags as labels')), - ('b', 'branches', None, _('annotate with branch names')), - ('', 'dots', None, _('use dots for runs')), - ('s', 'spaces', None, _('separate elements by spaces'))], - _('[OPTION]... [FILE [REV]...]'), - optionalrepo=True) -def debugdag(ui, repo, file_=None, *revs, **opts): - """format the changelog or an index DAG as a concise textual description - - If you pass a revlog index, the revlog's DAG is emitted. If you list - revision numbers, they get labeled in the output as rN. - - Otherwise, the changelog DAG of the current repo is emitted. - """ - spaces = opts.get('spaces') - dots = opts.get('dots') - if file_: - rlog = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), file_) - revs = set((int(r) for r in revs)) - def events(): - for r in rlog: - yield 'n', (r, list(p for p in rlog.parentrevs(r) - if p != -1)) - if r in revs: - yield 'l', (r, "r%i" % r) - elif repo: - cl = repo.changelog - tags = opts.get('tags') - branches = opts.get('branches') - if tags: - labels = {} - for l, n in repo.tags().items(): - labels.setdefault(cl.rev(n), []).append(l) - def events(): - b = "default" - for r in cl: - if branches: - newb = cl.read(cl.node(r))[5]['branch'] - if newb != b: - yield 'a', newb - b = newb - yield 'n', (r, list(p for p in cl.parentrevs(r) - if p != -1)) - if tags: - ls = labels.get(r) - if ls: - for l in ls: - yield 'l', (r, l) - else: - raise error.Abort(_('need repo for changelog dag')) - - for line in dagparser.dagtextlines(events(), - addspaces=spaces, - wraplabels=True, - wrapannotations=True, - wrapnonlinear=dots, - usedots=dots, - maxlinewidth=70): - ui.write(line) - ui.write("\n") - -@command('debugdata', debugrevlogopts, _('-c|-m|FILE REV')) -def debugdata(ui, repo, file_, rev=None, **opts): - """dump the contents of a data file revision""" - if opts.get('changelog') or opts.get('manifest') or opts.get('dir'): - if rev is not None: - raise error.CommandError('debugdata', _('invalid arguments')) - file_, rev = None, file_ - elif rev is None: - raise error.CommandError('debugdata', _('invalid arguments')) - r = cmdutil.openrevlog(repo, 'debugdata', file_, opts) - try: - ui.write(r.revision(r.lookup(rev))) - except KeyError: - raise error.Abort(_('invalid revision identifier %s') % rev) - -@command('debugdate', - [('e', 'extended', None, _('try extended date formats'))], - _('[-e] DATE [RANGE]'), - norepo=True, optionalrepo=True) -def debugdate(ui, date, range=None, **opts): - """parse and display a date""" - if opts["extended"]: - d = util.parsedate(date, util.extendeddateformats) - else: - d = util.parsedate(date) - ui.write(("internal: %s %s\n") % d) - ui.write(("standard: %s\n") % util.datestr(d)) - if range: - m = util.matchdate(range) - ui.write(("match: %s\n") % m(d[0])) - -@command('debugdiscovery', - [('', 'old', None, _('use old-style discovery')), - ('', 'nonheads', None, - _('use old-style discovery with non-heads included')), - ] + remoteopts, - _('[-l REV] [-r REV] [-b BRANCH]... [OTHER]')) -def debugdiscovery(ui, repo, remoteurl="default", **opts): - """runs the changeset discovery protocol in isolation""" - remoteurl, branches = hg.parseurl(ui.expandpath(remoteurl), - opts.get('branch')) - remote = hg.peer(repo, opts, remoteurl) - ui.status(_('comparing with %s\n') % util.hidepassword(remoteurl)) - - # make sure tests are repeatable - random.seed(12323) - - def doit(localheads, remoteheads, remote=remote): - if opts.get('old'): - if localheads: - raise error.Abort('cannot use localheads with old style ' - 'discovery') - if not util.safehasattr(remote, 'branches'): - # enable in-client legacy support - remote = localrepo.locallegacypeer(remote.local()) - common, _in, hds = treediscovery.findcommonincoming(repo, remote, - force=True) - common = set(common) - if not opts.get('nonheads'): - ui.write(("unpruned common: %s\n") % - " ".join(sorted(short(n) for n in common))) - dag = dagutil.revlogdag(repo.changelog) - all = dag.ancestorset(dag.internalizeall(common)) - common = dag.externalizeall(dag.headsetofconnecteds(all)) - else: - common, any, hds = setdiscovery.findcommonheads(ui, repo, remote) - common = set(common) - rheads = set(hds) - lheads = set(repo.heads()) - ui.write(("common heads: %s\n") % - " ".join(sorted(short(n) for n in common))) - if lheads <= common: - ui.write(("local is subset\n")) - elif rheads <= common: - ui.write(("remote is subset\n")) - - serverlogs = opts.get('serverlog') - if serverlogs: - for filename in serverlogs: - with open(filename, 'r') as logfile: - line = logfile.readline() - while line: - parts = line.strip().split(';') - op = parts[1] - if op == 'cg': - pass - elif op == 'cgss': - doit(parts[2].split(' '), parts[3].split(' ')) - elif op == 'unb': - doit(parts[3].split(' '), parts[2].split(' ')) - line = logfile.readline() - else: - remoterevs, _checkout = hg.addbranchrevs(repo, remote, branches, - opts.get('remote_head')) - localrevs = opts.get('local_head') - doit(localrevs, remoterevs) - -@command('debugextensions', formatteropts, [], norepo=True) -def debugextensions(ui, **opts): - '''show information about active extensions''' - exts = extensions.extensions(ui) - hgver = util.version() - fm = ui.formatter('debugextensions', opts) - for extname, extmod in sorted(exts, key=operator.itemgetter(0)): - isinternal = extensions.ismoduleinternal(extmod) - extsource = extmod.__file__ - if isinternal: - exttestedwith = [] # never expose magic string to users - else: - exttestedwith = getattr(extmod, 'testedwith', '').split() - extbuglink = getattr(extmod, 'buglink', None) - - fm.startitem() - - if ui.quiet or ui.verbose: - fm.write('name', '%s\n', extname) - else: - fm.write('name', '%s', extname) - if isinternal or hgver in exttestedwith: - fm.plain('\n') - elif not exttestedwith: - fm.plain(_(' (untested!)\n')) - else: - lasttestedversion = exttestedwith[-1] - fm.plain(' (%s!)\n' % lasttestedversion) - - fm.condwrite(ui.verbose and extsource, 'source', - _(' location: %s\n'), extsource or "") - - if ui.verbose: - fm.plain(_(' bundled: %s\n') % ['no', 'yes'][isinternal]) - fm.data(bundled=isinternal) - - fm.condwrite(ui.verbose and exttestedwith, 'testedwith', - _(' tested with: %s\n'), - fm.formatlist(exttestedwith, name='ver')) - - fm.condwrite(ui.verbose and extbuglink, 'buglink', - _(' bug reporting: %s\n'), extbuglink or "") - - fm.end() - -@command('debugfileset', - [('r', 'rev', '', _('apply the filespec on this revision'), _('REV'))], - _('[-r REV] FILESPEC')) -def debugfileset(ui, repo, expr, **opts): - '''parse and apply a fileset specification''' - ctx = scmutil.revsingle(repo, opts.get('rev'), None) - if ui.verbose: - tree = fileset.parse(expr) - ui.note(fileset.prettyformat(tree), "\n") - - for f in ctx.getfileset(expr): - ui.write("%s\n" % f) - -@command('debugfsinfo', [], _('[PATH]'), norepo=True) -def debugfsinfo(ui, path="."): - """show information detected about current filesystem""" - util.writefile('.debugfsinfo', '') - ui.write(('exec: %s\n') % (util.checkexec(path) and 'yes' or 'no')) - ui.write(('symlink: %s\n') % (util.checklink(path) and 'yes' or 'no')) - ui.write(('hardlink: %s\n') % (util.checknlink(path) and 'yes' or 'no')) - ui.write(('case-sensitive: %s\n') % (util.fscasesensitive('.debugfsinfo') - and 'yes' or 'no')) - os.unlink('.debugfsinfo') - -@command('debuggetbundle', - [('H', 'head', [], _('id of head node'), _('ID')), - ('C', 'common', [], _('id of common node'), _('ID')), - ('t', 'type', 'bzip2', _('bundle compression type to use'), _('TYPE'))], - _('REPO FILE [-H|-C ID]...'), - norepo=True) -def debuggetbundle(ui, repopath, bundlepath, head=None, common=None, **opts): - """retrieves a bundle from a repo - - Every ID must be a full-length hex node id string. Saves the bundle to the - given file. - """ - repo = hg.peer(ui, opts, repopath) - if not repo.capable('getbundle'): - raise error.Abort("getbundle() not supported by target repository") - args = {} - if common: - args['common'] = [bin(s) for s in common] - if head: - args['heads'] = [bin(s) for s in head] - # TODO: get desired bundlecaps from command line. - args['bundlecaps'] = None - bundle = repo.getbundle('debug', **args) - - bundletype = opts.get('type', 'bzip2').lower() - btypes = {'none': 'HG10UN', - 'bzip2': 'HG10BZ', - 'gzip': 'HG10GZ', - 'bundle2': 'HG20'} - bundletype = btypes.get(bundletype) - if bundletype not in bundle2.bundletypes: - raise error.Abort(_('unknown bundle type specified with --type')) - bundle2.writebundle(ui, bundle, bundlepath, bundletype) - -@command('debugignore', [], '[FILE]') -def debugignore(ui, repo, *files, **opts): - """display the combined ignore pattern and information about ignored files - - With no argument display the combined ignore pattern. - - Given space separated file names, shows if the given file is ignored and - if so, show the ignore rule (file and line number) that matched it. - """ - ignore = repo.dirstate._ignore - if not files: - # Show all the patterns - includepat = getattr(ignore, 'includepat', None) - if includepat is not None: - ui.write("%s\n" % includepat) - else: - raise error.Abort(_("no ignore patterns found")) - else: - for f in files: - nf = util.normpath(f) - ignored = None - ignoredata = None - if nf != '.': - if ignore(nf): - ignored = nf - ignoredata = repo.dirstate._ignorefileandline(nf) - else: - for p in util.finddirs(nf): - if ignore(p): - ignored = p - ignoredata = repo.dirstate._ignorefileandline(p) - break - if ignored: - if ignored == nf: - ui.write(_("%s is ignored\n") % f) - else: - ui.write(_("%s is ignored because of " - "containing folder %s\n") - % (f, ignored)) - ignorefile, lineno, line = ignoredata - ui.write(_("(ignore rule in %s, line %d: '%s')\n") - % (ignorefile, lineno, line)) - else: - ui.write(_("%s is not ignored\n") % f) - -@command('debugindex', debugrevlogopts + - [('f', 'format', 0, _('revlog format'), _('FORMAT'))], - _('[-f FORMAT] -c|-m|FILE'), - optionalrepo=True) -def debugindex(ui, repo, file_=None, **opts): - """dump the contents of an index file""" - r = cmdutil.openrevlog(repo, 'debugindex', file_, opts) - format = opts.get('format', 0) - if format not in (0, 1): - raise error.Abort(_("unknown format %d") % format) - - generaldelta = r.version & revlog.REVLOGGENERALDELTA - if generaldelta: - basehdr = ' delta' - else: - basehdr = ' base' - - if ui.debugflag: - shortfn = hex - else: - shortfn = short - - # There might not be anything in r, so have a sane default - idlen = 12 - for i in r: - idlen = len(shortfn(r.node(i))) - break - - if format == 0: - ui.write((" rev offset length " + basehdr + " linkrev" - " %s %s p2\n") % ("nodeid".ljust(idlen), "p1".ljust(idlen))) - elif format == 1: - ui.write((" rev flag offset length" - " size " + basehdr + " link p1 p2" - " %s\n") % "nodeid".rjust(idlen)) - - for i in r: - node = r.node(i) - if generaldelta: - base = r.deltaparent(i) - else: - base = r.chainbase(i) - if format == 0: - try: - pp = r.parents(node) - except Exception: - pp = [nullid, nullid] - ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % ( - i, r.start(i), r.length(i), base, r.linkrev(i), - shortfn(node), shortfn(pp[0]), shortfn(pp[1]))) - elif format == 1: - pr = r.parentrevs(i) - ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % ( - i, r.flags(i), r.start(i), r.length(i), r.rawsize(i), - base, r.linkrev(i), pr[0], pr[1], shortfn(node))) - -@command('debugindexdot', debugrevlogopts, - _('-c|-m|FILE'), optionalrepo=True) -def debugindexdot(ui, repo, file_=None, **opts): - """dump an index DAG as a graphviz dot file""" - r = cmdutil.openrevlog(repo, 'debugindexdot', file_, opts) - ui.write(("digraph G {\n")) - for i in r: - node = r.node(i) - pp = r.parents(node) - ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i)) - if pp[1] != nullid: - ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i)) - ui.write("}\n") - -@command('debugdeltachain', - debugrevlogopts + formatteropts, - _('-c|-m|FILE'), - optionalrepo=True) -def debugdeltachain(ui, repo, file_=None, **opts): - """dump information about delta chains in a revlog - - Output can be templatized. Available template keywords are: - - :``rev``: revision number - :``chainid``: delta chain identifier (numbered by unique base) - :``chainlen``: delta chain length to this revision - :``prevrev``: previous revision in delta chain - :``deltatype``: role of delta / how it was computed - :``compsize``: compressed size of revision - :``uncompsize``: uncompressed size of revision - :``chainsize``: total size of compressed revisions in chain - :``chainratio``: total chain size divided by uncompressed revision size - (new delta chains typically start at ratio 2.00) - :``lindist``: linear distance from base revision in delta chain to end - of this revision - :``extradist``: total size of revisions not part of this delta chain from - base of delta chain to end of this revision; a measurement - of how much extra data we need to read/seek across to read - the delta chain for this revision - :``extraratio``: extradist divided by chainsize; another representation of - how much unrelated data is needed to load this delta chain - """ - r = cmdutil.openrevlog(repo, 'debugdeltachain', file_, opts) - index = r.index - generaldelta = r.version & revlog.REVLOGGENERALDELTA - - def revinfo(rev): - e = index[rev] - compsize = e[1] - uncompsize = e[2] - chainsize = 0 - - if generaldelta: - if e[3] == e[5]: - deltatype = 'p1' - elif e[3] == e[6]: - deltatype = 'p2' - elif e[3] == rev - 1: - deltatype = 'prev' - elif e[3] == rev: - deltatype = 'base' - else: - deltatype = 'other' - else: - if e[3] == rev: - deltatype = 'base' - else: - deltatype = 'prev' - - chain = r._deltachain(rev)[0] - for iterrev in chain: - e = index[iterrev] - chainsize += e[1] - - return compsize, uncompsize, deltatype, chain, chainsize - - fm = ui.formatter('debugdeltachain', opts) - - fm.plain(' rev chain# chainlen prev delta ' - 'size rawsize chainsize ratio lindist extradist ' - 'extraratio\n') - - chainbases = {} - for rev in r: - comp, uncomp, deltatype, chain, chainsize = revinfo(rev) - chainbase = chain[0] - chainid = chainbases.setdefault(chainbase, len(chainbases) + 1) - basestart = r.start(chainbase) - revstart = r.start(rev) - lineardist = revstart + comp - basestart - extradist = lineardist - chainsize - try: - prevrev = chain[-2] - except IndexError: - prevrev = -1 - - chainratio = float(chainsize) / float(uncomp) - extraratio = float(extradist) / float(chainsize) - - fm.startitem() - fm.write('rev chainid chainlen prevrev deltatype compsize ' - 'uncompsize chainsize chainratio lindist extradist ' - 'extraratio', - '%7d %7d %8d %8d %7s %10d %10d %10d %9.5f %9d %9d %10.5f\n', - rev, chainid, len(chain), prevrev, deltatype, comp, - uncomp, chainsize, chainratio, lineardist, extradist, - extraratio, - rev=rev, chainid=chainid, chainlen=len(chain), - prevrev=prevrev, deltatype=deltatype, compsize=comp, - uncompsize=uncomp, chainsize=chainsize, - chainratio=chainratio, lindist=lineardist, - extradist=extradist, extraratio=extraratio) - - fm.end() - @command('debuginstall', [] + formatteropts, '', norepo=True) def debuginstall(ui, **opts): '''test Mercurial installation @@ -2750,6 +1942,16 @@ problems += 1 fm.condwrite(err, 'extensionserror', " %s\n", err) + compengines = util.compengines._engines.values() + fm.write('compengines', _('checking registered compression engines (%s)\n'), + fm.formatlist(sorted(e.name() for e in compengines), + name='compengine', fmt='%s', sep=', ')) + fm.write('compenginesavail', _('checking available compression engines ' + '(%s)\n'), + fm.formatlist(sorted(e.name() for e in compengines + if e.available()), + name='compengine', fmt='%s', sep=', ')) + # templates p = templater.templatepaths() fm.write('templatedirs', 'checking templates (%s)...\n', ' '.join(p)) @@ -3067,7 +2269,7 @@ with repo.lock(): n = repair.deleteobsmarkers(repo.obsstore, indices) - ui.write(_('deleted %i obsolescense markers\n') % n) + ui.write(_('deleted %i obsolescence markers\n') % n) return @@ -3153,14 +2355,14 @@ def complete(path, acceptable): dirstate = repo.dirstate - spec = os.path.normpath(os.path.join(os.getcwd(), path)) + spec = os.path.normpath(os.path.join(pycompat.getcwd(), path)) rootdir = repo.root + os.sep if spec != repo.root and not spec.startswith(rootdir): return [], [] if os.path.isdir(spec): spec += '/' spec = spec[len(rootdir):] - fixpaths = os.sep != '/' + fixpaths = pycompat.ossep != '/' if fixpaths: spec = spec.replace(os.sep, '/') speclen = len(spec) @@ -3366,6 +2568,8 @@ datasize = [None, 0, 0] fullsize = [None, 0, 0] deltasize = [None, 0, 0] + chunktypecounts = {} + chunktypesizes = {} def addsize(size, l): if l[0] is None or size < l[0]: @@ -3403,6 +2607,20 @@ elif delta != nullrev: numother += 1 + # Obtain data on the raw chunks in the revlog. + chunk = r._chunkraw(rev, rev)[1] + if chunk: + chunktype = chunk[0] + else: + chunktype = 'empty' + + if chunktype not in chunktypecounts: + chunktypecounts[chunktype] = 0 + chunktypesizes[chunktype] = 0 + + chunktypecounts[chunktype] += 1 + chunktypesizes[chunktype] += size + # Adjust size min value for empty cases for size in (datasize, fullsize, deltasize): if size[0] is None: @@ -3454,6 +2672,24 @@ ui.write((' full : ') + fmt % pcfmt(fulltotal, totalsize)) ui.write((' deltas : ') + fmt % pcfmt(deltatotal, totalsize)) + def fmtchunktype(chunktype): + if chunktype == 'empty': + return ' %s : ' % chunktype + elif chunktype in string.ascii_letters: + return ' 0x%s (%s) : ' % (hex(chunktype), chunktype) + else: + return ' 0x%s : ' % hex(chunktype) + + ui.write('\n') + ui.write(('chunks : ') + fmt2 % numrevs) + for chunktype in sorted(chunktypecounts): + ui.write(fmtchunktype(chunktype)) + ui.write(fmt % pcfmt(chunktypecounts[chunktype], numrevs)) + ui.write(('chunks size : ') + fmt2 % totalsize) + for chunktype in sorted(chunktypecounts): + ui.write(fmtchunktype(chunktype)) + ui.write(fmt % pcfmt(chunktypesizes[chunktype], totalsize)) + ui.write('\n') fmt = dfmtstr(max(avgchainlen, compratio)) ui.write(('avg chain length : ') + fmt % avgchainlen) @@ -3755,7 +2991,7 @@ if not items: return f = lambda fn: fn - if ui.configbool('ui', 'slash') and os.sep != '/': + if ui.configbool('ui', 'slash') and pycompat.ossep != '/': f = lambda fn: util.normpath(fn) fmt = 'f %%-%ds %%-%ds %%s' % ( max([len(abs) for abs in items]), @@ -5011,7 +4247,7 @@ lock = repo.lock() tr = repo.transaction('import') else: - dsguard = cmdutil.dirstateguard(repo, 'import') + dsguard = dirstateguard.dirstateguard(repo, 'import') parents = repo[None].parents() for patchurl in patches: if patchurl == '-': @@ -6574,11 +5810,8 @@ s = sshserver.sshserver(ui, repo) s.serve_forever() - if opts["cmdserver"]: - service = commandserver.createservice(ui, repo, opts) - else: - service = hgweb.createservice(ui, repo, opts) - return cmdutil.service(opts, initfn=service.init, runfn=service.run) + service = server.createservice(ui, repo, opts) + return server.runservice(opts, initfn=service.init, runfn=service.run) @command('^status|st', [('A', 'all', None, _('show status of all files')),
--- a/mercurial/commandserver.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/commandserver.py Sun Nov 27 20:44:52 2016 -0500 @@ -15,13 +15,13 @@ import signal import socket import struct -import sys import traceback from .i18n import _ from . import ( encoding, error, + pycompat, util, ) @@ -54,8 +54,8 @@ def write(self, data): if not data: return - self.out.write(struct.pack('>cI', self.channel, len(data))) - self.out.write(data) + # single write() to guarantee the same atomicity as the underlying file + self.out.write(struct.pack('>cI', self.channel, len(data)) + data) self.out.flush() def __getattr__(self, attr): @@ -153,7 +153,7 @@ based stream to fout. """ def __init__(self, ui, repo, fin, fout): - self.cwd = os.getcwd() + self.cwd = pycompat.getcwd() # developer config: cmdserver.log logpath = ui.config("cmdserver", "log", None) @@ -304,8 +304,8 @@ ui.flush() newfiles = [] nullfd = os.open(os.devnull, os.O_RDWR) - for f, sysf, mode in [(ui.fin, sys.stdin, 'rb'), - (ui.fout, sys.stdout, 'wb')]: + for f, sysf, mode in [(ui.fin, util.stdin, 'rb'), + (ui.fout, util.stdout, 'wb')]: if f is sysf: newfd = os.dup(f.fileno()) os.dup2(nullfd, f.fileno()) @@ -530,15 +530,3 @@ _serverequest(self.ui, self.repo, conn, h.createcmdserver) finally: gc.collect() # trigger __del__ since worker process uses os._exit - -_servicemap = { - 'pipe': pipeservice, - 'unix': unixforkingservice, - } - -def createservice(ui, repo, opts): - mode = opts['cmdserver'] - try: - return _servicemap[mode](ui, repo, opts) - except KeyError: - raise error.Abort(_('unknown mode %s') % mode)
--- a/mercurial/config.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/config.py Sun Nov 27 20:44:52 2016 -0500 @@ -90,13 +90,13 @@ self._source.pop((section, item), None) def parse(self, src, data, sections=None, remap=None, include=None): - sectionre = util.re.compile(r'\[([^\[]+)\]') - itemre = util.re.compile(r'([^=\s][^=]*?)\s*=\s*(.*\S|)') - contre = util.re.compile(r'\s+(\S|\S.*\S)\s*$') - emptyre = util.re.compile(r'(;|#|\s*$)') - commentre = util.re.compile(r'(;|#)') - unsetre = util.re.compile(r'%unset\s+(\S+)') - includere = util.re.compile(r'%include\s+(\S|\S.*\S)\s*$') + sectionre = util.re.compile(br'\[([^\[]+)\]') + itemre = util.re.compile(br'([^=\s][^=]*?)\s*=\s*(.*\S|)') + contre = util.re.compile(br'\s+(\S|\S.*\S)\s*$') + emptyre = util.re.compile(br'(;|#|\s*$)') + commentre = util.re.compile(br'(;|#)') + unsetre = util.re.compile(br'%unset\s+(\S+)') + includere = util.re.compile(br'%include\s+(\S|\S.*\S)\s*$') section = "" item = None line = 0
--- a/mercurial/context.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/context.py Sun Nov 27 20:44:52 2016 -0500 @@ -14,8 +14,11 @@ from .i18n import _ from .node import ( + addednodeid, bin, hex, + modifiednodeid, + newnodeid, nullid, nullrev, short, @@ -39,11 +42,6 @@ propertycache = util.propertycache -# Phony node value to stand-in for new files in some uses of -# manifests. Manifests support 21-byte hashes for nodes which are -# dirty in the working copy. -_newnode = '!' * 21 - nonascii = re.compile(r'[^\x21-\x7f]').search class basectx(object): @@ -142,7 +140,7 @@ removed.append(fn) elif flag1 != flag2: modified.append(fn) - elif node2 != _newnode: + elif node2 != newnodeid: # When comparing files between two commits, we save time by # not comparing the file contents when the nodeids differ. # Note that this means we incorrectly report a reverted change @@ -178,6 +176,8 @@ return hex(self.node()) def manifest(self): return self._manifest + def manifestctx(self): + return self._manifestctx def repo(self): return self._repo def phasestr(self): @@ -259,8 +259,10 @@ if path in self._manifestdelta: return (self._manifestdelta[path], self._manifestdelta.flags(path)) - node, flag = self._repo.manifest.find(self._changeset.manifest, path) - if not node: + mfl = self._repo.manifestlog + try: + node, flag = mfl[self._changeset.manifest].find(path) + except KeyError: raise error.ManifestLookupError(self._node, path, _('not found in manifest')) @@ -528,12 +530,15 @@ @propertycache def _manifest(self): - return self._repo.manifestlog[self._changeset.manifest].read() + return self._manifestctx.read() + + @propertycache + def _manifestctx(self): + return self._repo.manifestlog[self._changeset.manifest] @propertycache def _manifestdelta(self): - mfnode = self._changeset.manifest - return self._repo.manifestlog[mfnode].readdelta() + return self._manifestctx.readdelta() @propertycache def _parents(self): @@ -680,8 +685,7 @@ elif '_descendantrev' in self.__dict__: # this file context was created from a revision with a known # descendant, we can (lazily) correct for linkrev aliases - return self._adjustlinkrev(self._path, self._filelog, - self._filenode, self._descendantrev) + return self._adjustlinkrev(self._descendantrev) else: return self._filelog.linkrev(self._filerev) @@ -709,7 +713,10 @@ return False def __str__(self): - return "%s@%s" % (self.path(), self._changectx) + try: + return "%s@%s" % (self.path(), self._changectx) + except error.LookupError: + return "%s@???" % self.path() def __repr__(self): return "<%s %s>" % (type(self).__name__, str(self)) @@ -808,17 +815,13 @@ return True - def _adjustlinkrev(self, path, filelog, fnode, srcrev, inclusive=False): + def _adjustlinkrev(self, srcrev, inclusive=False): """return the first ancestor of <srcrev> introducing <fnode> If the linkrev of the file revision does not point to an ancestor of srcrev, we'll walk down the ancestors until we find one introducing this file revision. - :repo: a localrepository object (used to access changelog and manifest) - :path: the file path - :fnode: the nodeid of the file revision - :filelog: the filelog of this path :srcrev: the changeset revision we search ancestors from :inclusive: if true, the src revision will also be checked """ @@ -826,8 +829,7 @@ cl = repo.unfiltered().changelog mfl = repo.manifestlog # fetch the linkrev - fr = filelog.rev(fnode) - lkr = filelog.linkrev(fr) + lkr = self.linkrev() # hack to reuse ancestor computation when searching for renames memberanc = getattr(self, '_ancestrycontext', None) iteranc = None @@ -844,6 +846,8 @@ if lkr not in memberanc: if iteranc is None: iteranc = cl.ancestors(revs, lkr, inclusive=inclusive) + fnode = self._filenode + path = self._path for a in iteranc: ac = cl.read(a) # get changeset data (we avoid object creation) if path in ac[3]: # checking the 'files' field. @@ -871,8 +875,7 @@ noctx = not ('_changeid' in attrs or '_changectx' in attrs) if noctx or self.rev() == lkr: return self.linkrev() - return self._adjustlinkrev(self._path, self._filelog, self._filenode, - self.rev(), inclusive=True) + return self._adjustlinkrev(self.rev(), inclusive=True) def _parentfilectx(self, path, fileid, filelog): """create parent filectx keeping ancestry info for _adjustlinkrev()""" @@ -1231,23 +1234,13 @@ """ parents = self.parents() - man1 = parents[0].manifest() - man = man1.copy() - if len(parents) > 1: - man2 = self.p2().manifest() - def getman(f): - if f in man1: - return man1 - return man2 - else: - getman = lambda f: man1 + man = parents[0].manifest().copy() - copied = self._repo.dirstate.copies() ff = self._flagfunc - for i, l in (("a", self._status.added), ("m", self._status.modified)): + for i, l in ((addednodeid, self._status.added), + (modifiednodeid, self._status.modified)): for f in l: - orig = copied.get(f, f) - man[f] = getman(orig).get(orig, nullid) + i + man[f] = i try: man.setflag(f, ff(f)) except OSError: @@ -1582,7 +1575,7 @@ """ mf = self._repo['.']._manifestmatches(match, s) for f in s.modified + s.added: - mf[f] = _newnode + mf[f] = newnodeid mf.setflag(f, self.flags(f)) for f in s.removed: if f in mf:
--- a/mercurial/copies.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/copies.py Sun Nov 27 20:44:52 2016 -0500 @@ -278,7 +278,7 @@ ac = repo.changelog.ancestors(revs, inclusive=True) ctx._ancestrycontext = ac def makectx(f, n): - if len(n) != 20: # in a working context? + if n in node.wdirnodes: # in a working context? if ctx.rev() is None: return ctx.filectx(f) return repo[None][f]
--- a/mercurial/crecord.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/crecord.py Sun Nov 27 20:44:52 2016 -0500 @@ -14,14 +14,13 @@ import os import re import signal -import struct -import sys from .i18n import _ from . import ( encoding, error, patch as patchmod, + scmutil, util, ) stringio = util.stringio @@ -52,11 +51,7 @@ try: import curses - import fcntl - import termios curses.error - fcntl.ioctl - termios.TIOCGWINSZ except ImportError: # I have no idea if wcurses works with crecord... try: @@ -75,8 +70,6 @@ """ return curses and ui.interface("chunkselector") == "curses" -_origstdout = sys.__stdout__ # used by gethw() - class patchnode(object): """abstract class for patch graph nodes (i.e. patchroot, header, hunk, hunkline) @@ -440,7 +433,7 @@ def __repr__(self): return '<hunk %r@%d>' % (self.filename(), self.fromline) -def filterpatch(ui, chunks, chunkselector): +def filterpatch(ui, chunks, chunkselector, operation=None): """interactively filter patch chunks into applied-only chunks""" chunks = list(chunks) # convert chunks list into structure suitable for displaying/modifying @@ -453,7 +446,7 @@ uiheaders = [uiheader(h) for h in headers] # let user choose headers/hunks/lines, and mark their applied flags # accordingly - ret = chunkselector(ui, uiheaders) + ret = chunkselector(ui, uiheaders, operation=operation) appliedhunklist = [] for hdr in uiheaders: if (hdr.applied and @@ -473,25 +466,13 @@ return (appliedhunklist, ret) -def gethw(): - """ - magically get the current height and width of the window (without initscr) - - this is a rip-off of a rip-off - taken from the bpython code. it is - useful / necessary because otherwise curses.initscr() must be called, - which can leave the terminal in a nasty state after exiting. - """ - h, w = struct.unpack( - "hhhh", fcntl.ioctl(_origstdout, termios.TIOCGWINSZ, "\000"*8))[0:2] - return h, w - -def chunkselector(ui, headerlist): +def chunkselector(ui, headerlist, operation=None): """ curses interface to get selection of chunks, and mark the applied flags of the chosen chunks. """ ui.write(_('starting interactive selection\n')) - chunkselector = curseschunkselector(headerlist, ui) + chunkselector = curseschunkselector(headerlist, ui, operation) f = signal.getsignal(signal.SIGTSTP) curses.wrapper(chunkselector.main) if chunkselector.initerr is not None: @@ -505,12 +486,12 @@ return f(testfn, *args, **kwargs) return u -def testchunkselector(testfn, ui, headerlist): +def testchunkselector(testfn, ui, headerlist, operation=None): """ test interface to get selection of chunks, and mark the applied flags of the chosen chunks. """ - chunkselector = curseschunkselector(headerlist, ui) + chunkselector = curseschunkselector(headerlist, ui, operation) if testfn and os.path.exists(testfn): testf = open(testfn) testcommands = map(lambda x: x.rstrip('\n'), testf.readlines()) @@ -521,7 +502,7 @@ return chunkselector.opts class curseschunkselector(object): - def __init__(self, headerlist, ui): + def __init__(self, headerlist, ui, operation=None): # put the headers into a patch object self.headerlist = patch(headerlist) @@ -575,6 +556,9 @@ # if the last 'toggle all' command caused all changes to be applied self.waslasttoggleallapplied = True + # affects some ui text + self.operation = operation + def uparrowevent(self): """ try to select the previous item to the current item that has the @@ -976,10 +960,10 @@ pairname='legend') self.statuswin.refresh() return - line1 = ("SELECT CHUNKS: (j/k/up/dn/pgup/pgdn) move cursor; " - "(space/A) toggle hunk/all; (e)dit hunk;") - line2 = (" (f)old/unfold; (c)onfirm applied; (q)uit; (?) help " - "| [X]=hunk applied **=folded, toggle [a]mend mode") + line1 = _("SELECT CHUNKS: (j/k/up/dn/pgup/pgdn) move cursor; " + "(space/A) toggle hunk/all; (e)dit hunk;") + line2 = _(" (f)old/unfold; (c)onfirm applied; (q)uit; (?) help " + "| [X]=hunk applied **=folded, toggle [a]mend mode") printstring(self.statuswin, util.ellipsis(line1, self.xscreensize - 1), @@ -1259,7 +1243,7 @@ "handle window resizing" try: curses.endwin() - self.yscreensize, self.xscreensize = gethw() + self.xscreensize, self.yscreensize = scmutil.termsize(self.ui) self.statuswin.resize(self.numstatuslines, self.xscreensize) self.numpadlines = self.getnumlinesdisplayed(ignorefolding=True) + 1 self.chunkpad = curses.newpad(self.numpadlines, self.xscreensize) @@ -1321,7 +1305,8 @@ def helpwindow(self): "print a help window to the screen. exit after any keypress." - helptext = """ [press any key to return to the patch-display] + helptext = _( + """ [press any key to return to the patch-display] crecord allows you to interactively choose among the changes you have made, and confirm only those changes you select for further processing by the command @@ -1345,7 +1330,7 @@ c : confirm selected changes r : review/edit and confirm selected changes q : quit without confirming (no changes will be made) - ? : help (what you're currently reading)""" + ? : help (what you're currently reading)""") helpwin = curses.newwin(self.yscreensize, 0, 0, 0) helplines = helptext.split("\n") @@ -1384,7 +1369,7 @@ def reviewcommit(self): """ask for 'y' to be pressed to confirm selected. return True if confirmed.""" - confirmtext = ( + confirmtext = _( """if you answer yes to the following, the your currently chosen patch chunks will be loaded into an editor. you may modify the patch from the editor, and save the changes if you wish to change the patch. otherwise, you can just @@ -1416,19 +1401,19 @@ except ValueError: ver = 1 if ver < 2.19: - msg = ("The amend option is unavailable with hg versions < 2.2\n\n" - "Press any key to continue.") + msg = _("The amend option is unavailable with hg versions < 2.2\n\n" + "Press any key to continue.") elif opts.get('amend') is None: opts['amend'] = True - msg = ("Amend option is turned on -- commiting the currently " - "selected changes will not create a new changeset, but " - "instead update the most recently committed changeset.\n\n" - "Press any key to continue.") + msg = _("Amend option is turned on -- committing the currently " + "selected changes will not create a new changeset, but " + "instead update the most recently committed changeset.\n\n" + "Press any key to continue.") elif opts.get('amend') is True: opts['amend'] = None - msg = ("Amend option is turned off -- commiting the currently " - "selected changes will create a new changeset.\n\n" - "Press any key to continue.") + msg = _("Amend option is turned off -- committing the currently " + "selected changes will create a new changeset.\n\n" + "Press any key to continue.") if not test: self.confirmationwindow(msg) @@ -1629,7 +1614,7 @@ except curses.error: self.initerr = _('this diff is too large to be displayed') return - # initialize selecteitemendline (initial start-line is 0) + # initialize selecteditemendline (initial start-line is 0) self.selecteditemendline = self.getnumlinesdisplayed( self.currentselecteditem, recursechildren=False)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/debugcommands.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,851 @@ +# debugcommands.py - command processing for debug* commands +# +# Copyright 2005-2016 Matt Mackall <mpm@selenic.com> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import operator +import os +import random + +from .i18n import _ +from .node import ( + bin, + hex, + nullid, + short, +) +from . import ( + bundle2, + changegroup, + cmdutil, + commands, + context, + dagparser, + dagutil, + error, + exchange, + extensions, + fileset, + hg, + localrepo, + lock as lockmod, + pycompat, + revlog, + scmutil, + setdiscovery, + simplemerge, + streamclone, + treediscovery, + util, +) + +release = lockmod.release + +# We reuse the command table from commands because it is easier than +# teaching dispatch about multiple tables. +command = cmdutil.command(commands.table) + +@command('debugancestor', [], _('[INDEX] REV1 REV2'), optionalrepo=True) +def debugancestor(ui, repo, *args): + """find the ancestor revision of two revisions in a given index""" + if len(args) == 3: + index, rev1, rev2 = args + r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), index) + lookup = r.lookup + elif len(args) == 2: + if not repo: + raise error.Abort(_('there is no Mercurial repository here ' + '(.hg not found)')) + rev1, rev2 = args + r = repo.changelog + lookup = repo.lookup + else: + raise error.Abort(_('either two or three arguments required')) + a = r.ancestor(lookup(rev1), lookup(rev2)) + ui.write('%d:%s\n' % (r.rev(a), hex(a))) + +@command('debugapplystreamclonebundle', [], 'FILE') +def debugapplystreamclonebundle(ui, repo, fname): + """apply a stream clone bundle file""" + f = hg.openpath(ui, fname) + gen = exchange.readbundle(ui, f, fname) + gen.apply(repo) + +@command('debugbuilddag', + [('m', 'mergeable-file', None, _('add single file mergeable changes')), + ('o', 'overwritten-file', None, _('add single file all revs overwrite')), + ('n', 'new-file', None, _('add new file at each rev'))], + _('[OPTION]... [TEXT]')) +def debugbuilddag(ui, repo, text=None, + mergeable_file=False, + overwritten_file=False, + new_file=False): + """builds a repo with a given DAG from scratch in the current empty repo + + The description of the DAG is read from stdin if not given on the + command line. + + Elements: + + - "+n" is a linear run of n nodes based on the current default parent + - "." is a single node based on the current default parent + - "$" resets the default parent to null (implied at the start); + otherwise the default parent is always the last node created + - "<p" sets the default parent to the backref p + - "*p" is a fork at parent p, which is a backref + - "*p1/p2" is a merge of parents p1 and p2, which are backrefs + - "/p2" is a merge of the preceding node and p2 + - ":tag" defines a local tag for the preceding node + - "@branch" sets the named branch for subsequent nodes + - "#...\\n" is a comment up to the end of the line + + Whitespace between the above elements is ignored. + + A backref is either + + - a number n, which references the node curr-n, where curr is the current + node, or + - the name of a local tag you placed earlier using ":tag", or + - empty to denote the default parent. + + All string valued-elements are either strictly alphanumeric, or must + be enclosed in double quotes ("..."), with "\\" as escape character. + """ + + if text is None: + ui.status(_("reading DAG from stdin\n")) + text = ui.fin.read() + + cl = repo.changelog + if len(cl) > 0: + raise error.Abort(_('repository is not empty')) + + # determine number of revs in DAG + total = 0 + for type, data in dagparser.parsedag(text): + if type == 'n': + total += 1 + + if mergeable_file: + linesperrev = 2 + # make a file with k lines per rev + initialmergedlines = [str(i) for i in xrange(0, total * linesperrev)] + initialmergedlines.append("") + + tags = [] + + wlock = lock = tr = None + try: + wlock = repo.wlock() + lock = repo.lock() + tr = repo.transaction("builddag") + + at = -1 + atbranch = 'default' + nodeids = [] + id = 0 + ui.progress(_('building'), id, unit=_('revisions'), total=total) + for type, data in dagparser.parsedag(text): + if type == 'n': + ui.note(('node %s\n' % str(data))) + id, ps = data + + files = [] + fctxs = {} + + p2 = None + if mergeable_file: + fn = "mf" + p1 = repo[ps[0]] + if len(ps) > 1: + p2 = repo[ps[1]] + pa = p1.ancestor(p2) + base, local, other = [x[fn].data() for x in (pa, p1, + p2)] + m3 = simplemerge.Merge3Text(base, local, other) + ml = [l.strip() for l in m3.merge_lines()] + ml.append("") + elif at > 0: + ml = p1[fn].data().split("\n") + else: + ml = initialmergedlines + ml[id * linesperrev] += " r%i" % id + mergedtext = "\n".join(ml) + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, mergedtext) + + if overwritten_file: + fn = "of" + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) + + if new_file: + fn = "nf%i" % id + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) + if len(ps) > 1: + if not p2: + p2 = repo[ps[1]] + for fn in p2: + if fn.startswith("nf"): + files.append(fn) + fctxs[fn] = p2[fn] + + def fctxfn(repo, cx, path): + return fctxs.get(path) + + if len(ps) == 0 or ps[0] < 0: + pars = [None, None] + elif len(ps) == 1: + pars = [nodeids[ps[0]], None] + else: + pars = [nodeids[p] for p in ps] + cx = context.memctx(repo, pars, "r%i" % id, files, fctxfn, + date=(id, 0), + user="debugbuilddag", + extra={'branch': atbranch}) + nodeid = repo.commitctx(cx) + nodeids.append(nodeid) + at = id + elif type == 'l': + id, name = data + ui.note(('tag %s\n' % name)) + tags.append("%s %s\n" % (hex(repo.changelog.node(id)), name)) + elif type == 'a': + ui.note(('branch %s\n' % data)) + atbranch = data + ui.progress(_('building'), id, unit=_('revisions'), total=total) + tr.close() + + if tags: + repo.vfs.write("localtags", "".join(tags)) + finally: + ui.progress(_('building'), None) + release(tr, lock, wlock) + +def _debugchangegroup(ui, gen, all=None, indent=0, **opts): + indent_string = ' ' * indent + if all: + ui.write(("%sformat: id, p1, p2, cset, delta base, len(delta)\n") + % indent_string) + + def showchunks(named): + ui.write("\n%s%s\n" % (indent_string, named)) + chain = None + for chunkdata in iter(lambda: gen.deltachunk(chain), {}): + node = chunkdata['node'] + p1 = chunkdata['p1'] + p2 = chunkdata['p2'] + cs = chunkdata['cs'] + deltabase = chunkdata['deltabase'] + delta = chunkdata['delta'] + ui.write("%s%s %s %s %s %s %s\n" % + (indent_string, hex(node), hex(p1), hex(p2), + hex(cs), hex(deltabase), len(delta))) + chain = node + + chunkdata = gen.changelogheader() + showchunks("changelog") + chunkdata = gen.manifestheader() + showchunks("manifest") + for chunkdata in iter(gen.filelogheader, {}): + fname = chunkdata['filename'] + showchunks(fname) + else: + if isinstance(gen, bundle2.unbundle20): + raise error.Abort(_('use debugbundle2 for this file')) + chunkdata = gen.changelogheader() + chain = None + for chunkdata in iter(lambda: gen.deltachunk(chain), {}): + node = chunkdata['node'] + ui.write("%s%s\n" % (indent_string, hex(node))) + chain = node + +def _debugbundle2(ui, gen, all=None, **opts): + """lists the contents of a bundle2""" + if not isinstance(gen, bundle2.unbundle20): + raise error.Abort(_('not a bundle2 file')) + ui.write(('Stream params: %s\n' % repr(gen.params))) + for part in gen.iterparts(): + ui.write('%s -- %r\n' % (part.type, repr(part.params))) + if part.type == 'changegroup': + version = part.params.get('version', '01') + cg = changegroup.getunbundler(version, part, 'UN') + _debugchangegroup(ui, cg, all=all, indent=4, **opts) + +@command('debugbundle', + [('a', 'all', None, _('show all details')), + ('', 'spec', None, _('print the bundlespec of the bundle'))], + _('FILE'), + norepo=True) +def debugbundle(ui, bundlepath, all=None, spec=None, **opts): + """lists the contents of a bundle""" + with hg.openpath(ui, bundlepath) as f: + if spec: + spec = exchange.getbundlespec(ui, f) + ui.write('%s\n' % spec) + return + + gen = exchange.readbundle(ui, f, bundlepath) + if isinstance(gen, bundle2.unbundle20): + return _debugbundle2(ui, gen, all=all, **opts) + _debugchangegroup(ui, gen, all=all, **opts) + +@command('debugcheckstate', [], '') +def debugcheckstate(ui, repo): + """validate the correctness of the current dirstate""" + parent1, parent2 = repo.dirstate.parents() + m1 = repo[parent1].manifest() + m2 = repo[parent2].manifest() + errors = 0 + for f in repo.dirstate: + state = repo.dirstate[f] + if state in "nr" and f not in m1: + ui.warn(_("%s in state %s, but not in manifest1\n") % (f, state)) + errors += 1 + if state in "a" and f in m1: + ui.warn(_("%s in state %s, but also in manifest1\n") % (f, state)) + errors += 1 + if state in "m" and f not in m1 and f not in m2: + ui.warn(_("%s in state %s, but not in either manifest\n") % + (f, state)) + errors += 1 + for f in m1: + state = repo.dirstate[f] + if state not in "nrm": + ui.warn(_("%s in manifest1, but listed as state %s") % (f, state)) + errors += 1 + if errors: + error = _(".hg/dirstate inconsistent with current parent's manifest") + raise error.Abort(error) + +@command('debugcommands', [], _('[COMMAND]'), norepo=True) +def debugcommands(ui, cmd='', *args): + """list all available commands and options""" + for cmd, vals in sorted(commands.table.iteritems()): + cmd = cmd.split('|')[0].strip('^') + opts = ', '.join([i[1] for i in vals[1]]) + ui.write('%s: %s\n' % (cmd, opts)) + +@command('debugcomplete', + [('o', 'options', None, _('show the command options'))], + _('[-o] CMD'), + norepo=True) +def debugcomplete(ui, cmd='', **opts): + """returns the completion list associated with the given command""" + + if opts.get('options'): + options = [] + otables = [commands.globalopts] + if cmd: + aliases, entry = cmdutil.findcmd(cmd, commands.table, False) + otables.append(entry[1]) + for t in otables: + for o in t: + if "(DEPRECATED)" in o[3]: + continue + if o[0]: + options.append('-%s' % o[0]) + options.append('--%s' % o[1]) + ui.write("%s\n" % "\n".join(options)) + return + + cmdlist, unused_allcmds = cmdutil.findpossible(cmd, commands.table) + if ui.verbose: + cmdlist = [' '.join(c[0]) for c in cmdlist.values()] + ui.write("%s\n" % "\n".join(sorted(cmdlist))) + +@command('debugcreatestreamclonebundle', [], 'FILE') +def debugcreatestreamclonebundle(ui, repo, fname): + """create a stream clone bundle file + + Stream bundles are special bundles that are essentially archives of + revlog files. They are commonly used for cloning very quickly. + """ + requirements, gen = streamclone.generatebundlev1(repo) + changegroup.writechunks(ui, gen, fname) + + ui.write(_('bundle requirements: %s\n') % ', '.join(sorted(requirements))) + +@command('debugdag', + [('t', 'tags', None, _('use tags as labels')), + ('b', 'branches', None, _('annotate with branch names')), + ('', 'dots', None, _('use dots for runs')), + ('s', 'spaces', None, _('separate elements by spaces'))], + _('[OPTION]... [FILE [REV]...]'), + optionalrepo=True) +def debugdag(ui, repo, file_=None, *revs, **opts): + """format the changelog or an index DAG as a concise textual description + + If you pass a revlog index, the revlog's DAG is emitted. If you list + revision numbers, they get labeled in the output as rN. + + Otherwise, the changelog DAG of the current repo is emitted. + """ + spaces = opts.get('spaces') + dots = opts.get('dots') + if file_: + rlog = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), + file_) + revs = set((int(r) for r in revs)) + def events(): + for r in rlog: + yield 'n', (r, list(p for p in rlog.parentrevs(r) + if p != -1)) + if r in revs: + yield 'l', (r, "r%i" % r) + elif repo: + cl = repo.changelog + tags = opts.get('tags') + branches = opts.get('branches') + if tags: + labels = {} + for l, n in repo.tags().items(): + labels.setdefault(cl.rev(n), []).append(l) + def events(): + b = "default" + for r in cl: + if branches: + newb = cl.read(cl.node(r))[5]['branch'] + if newb != b: + yield 'a', newb + b = newb + yield 'n', (r, list(p for p in cl.parentrevs(r) + if p != -1)) + if tags: + ls = labels.get(r) + if ls: + for l in ls: + yield 'l', (r, l) + else: + raise error.Abort(_('need repo for changelog dag')) + + for line in dagparser.dagtextlines(events(), + addspaces=spaces, + wraplabels=True, + wrapannotations=True, + wrapnonlinear=dots, + usedots=dots, + maxlinewidth=70): + ui.write(line) + ui.write("\n") + +@command('debugdata', commands.debugrevlogopts, _('-c|-m|FILE REV')) +def debugdata(ui, repo, file_, rev=None, **opts): + """dump the contents of a data file revision""" + if opts.get('changelog') or opts.get('manifest') or opts.get('dir'): + if rev is not None: + raise error.CommandError('debugdata', _('invalid arguments')) + file_, rev = None, file_ + elif rev is None: + raise error.CommandError('debugdata', _('invalid arguments')) + r = cmdutil.openrevlog(repo, 'debugdata', file_, opts) + try: + ui.write(r.revision(r.lookup(rev))) + except KeyError: + raise error.Abort(_('invalid revision identifier %s') % rev) + +@command('debugdate', + [('e', 'extended', None, _('try extended date formats'))], + _('[-e] DATE [RANGE]'), + norepo=True, optionalrepo=True) +def debugdate(ui, date, range=None, **opts): + """parse and display a date""" + if opts["extended"]: + d = util.parsedate(date, util.extendeddateformats) + else: + d = util.parsedate(date) + ui.write(("internal: %s %s\n") % d) + ui.write(("standard: %s\n") % util.datestr(d)) + if range: + m = util.matchdate(range) + ui.write(("match: %s\n") % m(d[0])) + +@command('debugdeltachain', + commands.debugrevlogopts + commands.formatteropts, + _('-c|-m|FILE'), + optionalrepo=True) +def debugdeltachain(ui, repo, file_=None, **opts): + """dump information about delta chains in a revlog + + Output can be templatized. Available template keywords are: + + :``rev``: revision number + :``chainid``: delta chain identifier (numbered by unique base) + :``chainlen``: delta chain length to this revision + :``prevrev``: previous revision in delta chain + :``deltatype``: role of delta / how it was computed + :``compsize``: compressed size of revision + :``uncompsize``: uncompressed size of revision + :``chainsize``: total size of compressed revisions in chain + :``chainratio``: total chain size divided by uncompressed revision size + (new delta chains typically start at ratio 2.00) + :``lindist``: linear distance from base revision in delta chain to end + of this revision + :``extradist``: total size of revisions not part of this delta chain from + base of delta chain to end of this revision; a measurement + of how much extra data we need to read/seek across to read + the delta chain for this revision + :``extraratio``: extradist divided by chainsize; another representation of + how much unrelated data is needed to load this delta chain + """ + r = cmdutil.openrevlog(repo, 'debugdeltachain', file_, opts) + index = r.index + generaldelta = r.version & revlog.REVLOGGENERALDELTA + + def revinfo(rev): + e = index[rev] + compsize = e[1] + uncompsize = e[2] + chainsize = 0 + + if generaldelta: + if e[3] == e[5]: + deltatype = 'p1' + elif e[3] == e[6]: + deltatype = 'p2' + elif e[3] == rev - 1: + deltatype = 'prev' + elif e[3] == rev: + deltatype = 'base' + else: + deltatype = 'other' + else: + if e[3] == rev: + deltatype = 'base' + else: + deltatype = 'prev' + + chain = r._deltachain(rev)[0] + for iterrev in chain: + e = index[iterrev] + chainsize += e[1] + + return compsize, uncompsize, deltatype, chain, chainsize + + fm = ui.formatter('debugdeltachain', opts) + + fm.plain(' rev chain# chainlen prev delta ' + 'size rawsize chainsize ratio lindist extradist ' + 'extraratio\n') + + chainbases = {} + for rev in r: + comp, uncomp, deltatype, chain, chainsize = revinfo(rev) + chainbase = chain[0] + chainid = chainbases.setdefault(chainbase, len(chainbases) + 1) + basestart = r.start(chainbase) + revstart = r.start(rev) + lineardist = revstart + comp - basestart + extradist = lineardist - chainsize + try: + prevrev = chain[-2] + except IndexError: + prevrev = -1 + + chainratio = float(chainsize) / float(uncomp) + extraratio = float(extradist) / float(chainsize) + + fm.startitem() + fm.write('rev chainid chainlen prevrev deltatype compsize ' + 'uncompsize chainsize chainratio lindist extradist ' + 'extraratio', + '%7d %7d %8d %8d %7s %10d %10d %10d %9.5f %9d %9d %10.5f\n', + rev, chainid, len(chain), prevrev, deltatype, comp, + uncomp, chainsize, chainratio, lineardist, extradist, + extraratio, + rev=rev, chainid=chainid, chainlen=len(chain), + prevrev=prevrev, deltatype=deltatype, compsize=comp, + uncompsize=uncomp, chainsize=chainsize, + chainratio=chainratio, lindist=lineardist, + extradist=extradist, extraratio=extraratio) + + fm.end() + +@command('debugdiscovery', + [('', 'old', None, _('use old-style discovery')), + ('', 'nonheads', None, + _('use old-style discovery with non-heads included')), + ] + commands.remoteopts, + _('[-l REV] [-r REV] [-b BRANCH]... [OTHER]')) +def debugdiscovery(ui, repo, remoteurl="default", **opts): + """runs the changeset discovery protocol in isolation""" + remoteurl, branches = hg.parseurl(ui.expandpath(remoteurl), + opts.get('branch')) + remote = hg.peer(repo, opts, remoteurl) + ui.status(_('comparing with %s\n') % util.hidepassword(remoteurl)) + + # make sure tests are repeatable + random.seed(12323) + + def doit(localheads, remoteheads, remote=remote): + if opts.get('old'): + if localheads: + raise error.Abort('cannot use localheads with old style ' + 'discovery') + if not util.safehasattr(remote, 'branches'): + # enable in-client legacy support + remote = localrepo.locallegacypeer(remote.local()) + common, _in, hds = treediscovery.findcommonincoming(repo, remote, + force=True) + common = set(common) + if not opts.get('nonheads'): + ui.write(("unpruned common: %s\n") % + " ".join(sorted(short(n) for n in common))) + dag = dagutil.revlogdag(repo.changelog) + all = dag.ancestorset(dag.internalizeall(common)) + common = dag.externalizeall(dag.headsetofconnecteds(all)) + else: + common, any, hds = setdiscovery.findcommonheads(ui, repo, remote) + common = set(common) + rheads = set(hds) + lheads = set(repo.heads()) + ui.write(("common heads: %s\n") % + " ".join(sorted(short(n) for n in common))) + if lheads <= common: + ui.write(("local is subset\n")) + elif rheads <= common: + ui.write(("remote is subset\n")) + + serverlogs = opts.get('serverlog') + if serverlogs: + for filename in serverlogs: + with open(filename, 'r') as logfile: + line = logfile.readline() + while line: + parts = line.strip().split(';') + op = parts[1] + if op == 'cg': + pass + elif op == 'cgss': + doit(parts[2].split(' '), parts[3].split(' ')) + elif op == 'unb': + doit(parts[3].split(' '), parts[2].split(' ')) + line = logfile.readline() + else: + remoterevs, _checkout = hg.addbranchrevs(repo, remote, branches, + opts.get('remote_head')) + localrevs = opts.get('local_head') + doit(localrevs, remoterevs) + +@command('debugextensions', commands.formatteropts, [], norepo=True) +def debugextensions(ui, **opts): + '''show information about active extensions''' + exts = extensions.extensions(ui) + hgver = util.version() + fm = ui.formatter('debugextensions', opts) + for extname, extmod in sorted(exts, key=operator.itemgetter(0)): + isinternal = extensions.ismoduleinternal(extmod) + extsource = extmod.__file__ + if isinternal: + exttestedwith = [] # never expose magic string to users + else: + exttestedwith = getattr(extmod, 'testedwith', '').split() + extbuglink = getattr(extmod, 'buglink', None) + + fm.startitem() + + if ui.quiet or ui.verbose: + fm.write('name', '%s\n', extname) + else: + fm.write('name', '%s', extname) + if isinternal or hgver in exttestedwith: + fm.plain('\n') + elif not exttestedwith: + fm.plain(_(' (untested!)\n')) + else: + lasttestedversion = exttestedwith[-1] + fm.plain(' (%s!)\n' % lasttestedversion) + + fm.condwrite(ui.verbose and extsource, 'source', + _(' location: %s\n'), extsource or "") + + if ui.verbose: + fm.plain(_(' bundled: %s\n') % ['no', 'yes'][isinternal]) + fm.data(bundled=isinternal) + + fm.condwrite(ui.verbose and exttestedwith, 'testedwith', + _(' tested with: %s\n'), + fm.formatlist(exttestedwith, name='ver')) + + fm.condwrite(ui.verbose and extbuglink, 'buglink', + _(' bug reporting: %s\n'), extbuglink or "") + + fm.end() + +@command('debugfileset', + [('r', 'rev', '', _('apply the filespec on this revision'), _('REV'))], + _('[-r REV] FILESPEC')) +def debugfileset(ui, repo, expr, **opts): + '''parse and apply a fileset specification''' + ctx = scmutil.revsingle(repo, opts.get('rev'), None) + if ui.verbose: + tree = fileset.parse(expr) + ui.note(fileset.prettyformat(tree), "\n") + + for f in ctx.getfileset(expr): + ui.write("%s\n" % f) + +@command('debugfsinfo', [], _('[PATH]'), norepo=True) +def debugfsinfo(ui, path="."): + """show information detected about current filesystem""" + util.writefile('.debugfsinfo', '') + ui.write(('exec: %s\n') % (util.checkexec(path) and 'yes' or 'no')) + ui.write(('symlink: %s\n') % (util.checklink(path) and 'yes' or 'no')) + ui.write(('hardlink: %s\n') % (util.checknlink(path) and 'yes' or 'no')) + ui.write(('case-sensitive: %s\n') % (util.fscasesensitive('.debugfsinfo') + and 'yes' or 'no')) + os.unlink('.debugfsinfo') + +@command('debuggetbundle', + [('H', 'head', [], _('id of head node'), _('ID')), + ('C', 'common', [], _('id of common node'), _('ID')), + ('t', 'type', 'bzip2', _('bundle compression type to use'), _('TYPE'))], + _('REPO FILE [-H|-C ID]...'), + norepo=True) +def debuggetbundle(ui, repopath, bundlepath, head=None, common=None, **opts): + """retrieves a bundle from a repo + + Every ID must be a full-length hex node id string. Saves the bundle to the + given file. + """ + repo = hg.peer(ui, opts, repopath) + if not repo.capable('getbundle'): + raise error.Abort("getbundle() not supported by target repository") + args = {} + if common: + args['common'] = [bin(s) for s in common] + if head: + args['heads'] = [bin(s) for s in head] + # TODO: get desired bundlecaps from command line. + args['bundlecaps'] = None + bundle = repo.getbundle('debug', **args) + + bundletype = opts.get('type', 'bzip2').lower() + btypes = {'none': 'HG10UN', + 'bzip2': 'HG10BZ', + 'gzip': 'HG10GZ', + 'bundle2': 'HG20'} + bundletype = btypes.get(bundletype) + if bundletype not in bundle2.bundletypes: + raise error.Abort(_('unknown bundle type specified with --type')) + bundle2.writebundle(ui, bundle, bundlepath, bundletype) + +@command('debugignore', [], '[FILE]') +def debugignore(ui, repo, *files, **opts): + """display the combined ignore pattern and information about ignored files + + With no argument display the combined ignore pattern. + + Given space separated file names, shows if the given file is ignored and + if so, show the ignore rule (file and line number) that matched it. + """ + ignore = repo.dirstate._ignore + if not files: + # Show all the patterns + includepat = getattr(ignore, 'includepat', None) + if includepat is not None: + ui.write("%s\n" % includepat) + else: + raise error.Abort(_("no ignore patterns found")) + else: + for f in files: + nf = util.normpath(f) + ignored = None + ignoredata = None + if nf != '.': + if ignore(nf): + ignored = nf + ignoredata = repo.dirstate._ignorefileandline(nf) + else: + for p in util.finddirs(nf): + if ignore(p): + ignored = p + ignoredata = repo.dirstate._ignorefileandline(p) + break + if ignored: + if ignored == nf: + ui.write(_("%s is ignored\n") % f) + else: + ui.write(_("%s is ignored because of " + "containing folder %s\n") + % (f, ignored)) + ignorefile, lineno, line = ignoredata + ui.write(_("(ignore rule in %s, line %d: '%s')\n") + % (ignorefile, lineno, line)) + else: + ui.write(_("%s is not ignored\n") % f) + +@command('debugindex', commands.debugrevlogopts + + [('f', 'format', 0, _('revlog format'), _('FORMAT'))], + _('[-f FORMAT] -c|-m|FILE'), + optionalrepo=True) +def debugindex(ui, repo, file_=None, **opts): + """dump the contents of an index file""" + r = cmdutil.openrevlog(repo, 'debugindex', file_, opts) + format = opts.get('format', 0) + if format not in (0, 1): + raise error.Abort(_("unknown format %d") % format) + + generaldelta = r.version & revlog.REVLOGGENERALDELTA + if generaldelta: + basehdr = ' delta' + else: + basehdr = ' base' + + if ui.debugflag: + shortfn = hex + else: + shortfn = short + + # There might not be anything in r, so have a sane default + idlen = 12 + for i in r: + idlen = len(shortfn(r.node(i))) + break + + if format == 0: + ui.write((" rev offset length " + basehdr + " linkrev" + " %s %s p2\n") % ("nodeid".ljust(idlen), "p1".ljust(idlen))) + elif format == 1: + ui.write((" rev flag offset length" + " size " + basehdr + " link p1 p2" + " %s\n") % "nodeid".rjust(idlen)) + + for i in r: + node = r.node(i) + if generaldelta: + base = r.deltaparent(i) + else: + base = r.chainbase(i) + if format == 0: + try: + pp = r.parents(node) + except Exception: + pp = [nullid, nullid] + ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % ( + i, r.start(i), r.length(i), base, r.linkrev(i), + shortfn(node), shortfn(pp[0]), shortfn(pp[1]))) + elif format == 1: + pr = r.parentrevs(i) + ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % ( + i, r.flags(i), r.start(i), r.length(i), r.rawsize(i), + base, r.linkrev(i), pr[0], pr[1], shortfn(node))) + +@command('debugindexdot', commands.debugrevlogopts, + _('-c|-m|FILE'), optionalrepo=True) +def debugindexdot(ui, repo, file_=None, **opts): + """dump an index DAG as a graphviz dot file""" + r = cmdutil.openrevlog(repo, 'debugindexdot', file_, opts) + ui.write(("digraph G {\n")) + for i in r: + node = r.node(i) + pp = r.parents(node) + ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i)) + if pp[1] != nullid: + ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i)) + ui.write("}\n")
--- a/mercurial/destutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/destutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -133,7 +133,7 @@ assert node is not None, "'tip' exists even in empty repository" return node, movemark, None -# order in which each step should be evalutated +# order in which each step should be evaluated # steps are run until one finds a destination destupdatesteps = ['evolution', 'bookmark', 'branch', 'branchfallback'] # mapping to ease extension overriding steps.
--- a/mercurial/dirstate.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/dirstate.py Sun Nov 27 20:44:52 2016 -0500 @@ -21,6 +21,7 @@ osutil, parsers, pathutil, + pycompat, scmutil, util, ) @@ -215,7 +216,7 @@ @propertycache def _slash(self): - return self._ui.configbool('ui', 'slash') and os.sep != '/' + return self._ui.configbool('ui', 'slash') and pycompat.ossep != '/' @propertycache def _checklink(self): @@ -270,7 +271,7 @@ @propertycache def _cwd(self): - return os.getcwd() + return pycompat.getcwd() def getcwd(self): '''Return the path from which a canonical path is calculated.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/dirstateguard.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,69 @@ +# dirstateguard.py - class to allow restoring dirstate after failure +# +# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from .i18n import _ + +from . import ( + error, +) + +class dirstateguard(object): + '''Restore dirstate at unexpected failure. + + At the construction, this class does: + + - write current ``repo.dirstate`` out, and + - save ``.hg/dirstate`` into the backup file + + This restores ``.hg/dirstate`` from backup file, if ``release()`` + is invoked before ``close()``. + + This just removes the backup file at ``close()`` before ``release()``. + ''' + + def __init__(self, repo, name): + self._repo = repo + self._active = False + self._closed = False + self._suffix = '.backup.%s.%d' % (name, id(self)) + repo.dirstate.savebackup(repo.currenttransaction(), self._suffix) + self._active = True + + def __del__(self): + if self._active: # still active + # this may occur, even if this class is used correctly: + # for example, releasing other resources like transaction + # may raise exception before ``dirstateguard.release`` in + # ``release(tr, ....)``. + self._abort() + + def close(self): + if not self._active: # already inactivated + msg = (_("can't close already inactivated backup: dirstate%s") + % self._suffix) + raise error.Abort(msg) + + self._repo.dirstate.clearbackup(self._repo.currenttransaction(), + self._suffix) + self._active = False + self._closed = True + + def _abort(self): + self._repo.dirstate.restorebackup(self._repo.currenttransaction(), + self._suffix) + self._active = False + + def release(self): + if not self._closed: + if not self._active: # already inactivated + msg = (_("can't release already inactivated backup:" + " dirstate%s") + % self._suffix) + raise error.Abort(msg) + self._abort()
--- a/mercurial/dispatch.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/dispatch.py Sun Nov 27 20:44:52 2016 -0500 @@ -15,7 +15,6 @@ import re import shlex import signal -import socket import sys import time import traceback @@ -26,6 +25,7 @@ from . import ( cmdutil, commands, + debugcommands, demandimport, encoding, error, @@ -35,7 +35,9 @@ hg, hook, profiling, + pycompat, revset, + scmutil, templatefilters, templatekw, templater, @@ -57,7 +59,7 @@ def run(): "run the command in sys.argv" - sys.exit((dispatch(request(sys.argv[1:])) or 0) & 255) + sys.exit((dispatch(request(pycompat.sysargv[1:])) or 0) & 255) def _getsimilar(symbols, value): sim = lambda x: difflib.SequenceMatcher(None, value, x).ratio() @@ -95,7 +97,7 @@ elif req.ui: ferr = req.ui.ferr else: - ferr = sys.stderr + ferr = util.stderr try: if not req.ui: @@ -216,30 +218,15 @@ return callcatch(ui, _runcatchfunc) def callcatch(ui, func): - """call func() with global exception handling - - return func() if no exception happens. otherwise do some error handling - and return an exit code accordingly. + """like scmutil.callcatch but handles more high-level exceptions about + config parsing and commands. besides, use handlecommandexception to handle + uncaught exceptions. """ try: - return func() - # Global exception handling, alphabetically - # Mercurial-specific first, followed by built-in and library exceptions + return scmutil.callcatch(ui, func) except error.AmbiguousCommand as inst: ui.warn(_("hg: command '%s' is ambiguous:\n %s\n") % (inst.args[0], " ".join(inst.args[1]))) - except error.ParseError as inst: - _formatparse(ui.warn, inst) - return -1 - except error.LockHeld as inst: - if inst.errno == errno.ETIMEDOUT: - reason = _('timed out waiting for lock held by %s') % inst.locker - else: - reason = _('lock held by %s') % inst.locker - ui.warn(_("abort: %s: %s\n") % (inst.desc or inst.filename, reason)) - except error.LockUnavailable as inst: - ui.warn(_("abort: could not lock %s: %s\n") % - (inst.desc or inst.filename, inst.strerror)) except error.CommandError as inst: if inst.args[0]: ui.warn(_("hg %s: %s\n") % (inst.args[0], inst.args[1])) @@ -247,34 +234,9 @@ else: ui.warn(_("hg: %s\n") % inst.args[1]) commands.help_(ui, 'shortlist') - except error.OutOfBandError as inst: - if inst.args: - msg = _("abort: remote error:\n") - else: - msg = _("abort: remote error\n") - ui.warn(msg) - if inst.args: - ui.warn(''.join(inst.args)) - if inst.hint: - ui.warn('(%s)\n' % inst.hint) - except error.RepoError as inst: - ui.warn(_("abort: %s!\n") % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - except error.ResponseError as inst: - ui.warn(_("abort: %s") % inst.args[0]) - if not isinstance(inst.args[1], basestring): - ui.warn(" %r\n" % (inst.args[1],)) - elif not inst.args[1]: - ui.warn(_(" empty string\n")) - else: - ui.warn("\n%r\n" % util.ellipsis(inst.args[1])) - except error.CensoredNodeError as inst: - ui.warn(_("abort: file censored %s!\n") % inst) - except error.RevlogError as inst: - ui.warn(_("abort: %s!\n") % inst) - except error.SignalInterrupt: - ui.warn(_("killed!\n")) + except error.ParseError as inst: + _formatparse(ui.warn, inst) + return -1 except error.UnknownCommand as inst: ui.warn(_("hg: unknown command '%s'\n") % inst.args[0]) try: @@ -290,61 +252,11 @@ suggested = True if not suggested: commands.help_(ui, 'shortlist') - except error.InterventionRequired as inst: - ui.warn("%s\n" % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - return 1 - except error.Abort as inst: - ui.warn(_("abort: %s\n") % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - except ImportError as inst: - ui.warn(_("abort: %s!\n") % inst) - m = str(inst).split()[-1] - if m in "mpatch bdiff".split(): - ui.warn(_("(did you forget to compile extensions?)\n")) - elif m in "zlib".split(): - ui.warn(_("(is your Python install correct?)\n")) - except IOError as inst: - if util.safehasattr(inst, "code"): - ui.warn(_("abort: %s\n") % inst) - elif util.safehasattr(inst, "reason"): - try: # usually it is in the form (errno, strerror) - reason = inst.reason.args[1] - except (AttributeError, IndexError): - # it might be anything, for example a string - reason = inst.reason - if isinstance(reason, unicode): - # SSLError of Python 2.7.9 contains a unicode - reason = reason.encode(encoding.encoding, 'replace') - ui.warn(_("abort: error: %s\n") % reason) - elif (util.safehasattr(inst, "args") - and inst.args and inst.args[0] == errno.EPIPE): - pass - elif getattr(inst, "strerror", None): - if getattr(inst, "filename", None): - ui.warn(_("abort: %s: %s\n") % (inst.strerror, inst.filename)) - else: - ui.warn(_("abort: %s\n") % inst.strerror) - else: - raise - except OSError as inst: - if getattr(inst, "filename", None) is not None: - ui.warn(_("abort: %s: '%s'\n") % (inst.strerror, inst.filename)) - else: - ui.warn(_("abort: %s\n") % inst.strerror) + except IOError: + raise except KeyboardInterrupt: raise - except MemoryError: - ui.warn(_("abort: out of memory\n")) - except SystemExit as inst: - # Commands shouldn't sys.exit directly, but give a return code. - # Just in case catch this and and pass exit code to caller. - return inst.code - except socket.error as inst: - ui.warn(_("abort: %s\n") % inst.args[-1]) - except: # perhaps re-raises + except: # probably re-raises if not handlecommandexception(ui): raise @@ -665,7 +577,7 @@ """ if wd is None: try: - wd = os.getcwd() + wd = pycompat.getcwd() except OSError as e: raise error.Abort(_("error getting current working directory: %s") % e.strerror) @@ -712,14 +624,6 @@ return lambda: runcommand(lui, None, cmd, args[:1], ui, options, d, [], {}) -def _cmdattr(ui, cmd, func, attr): - try: - return getattr(func, attr) - except AttributeError: - ui.deprecwarn("missing attribute '%s', use @command decorator " - "to register '%s'" % (attr, cmd), '3.8') - return False - _loaded = set() # list of (objname, loadermod, loadername) tuple: @@ -768,6 +672,10 @@ # (reposetup is handled in hg.repository) + # Side-effect of accessing is debugcommands module is guaranteed to be + # imported and commands.table is populated. + debugcommands.command + addaliases(lui, commands.table) # All aliases and commands are completely defined, now. @@ -848,7 +756,7 @@ with profiling.maybeprofile(lui): repo = None cmdpats = args[:] - if not _cmdattr(ui, cmd, func, 'norepo'): + if not func.norepo: # use the repo from the request only if we don't have -R if not rpath and not cwd: repo = req.repo @@ -871,9 +779,8 @@ except error.RepoError: if rpath and rpath[-1]: # invalid -R path raise - if not _cmdattr(ui, cmd, func, 'optionalrepo'): - if (_cmdattr(ui, cmd, func, 'inferrepo') and - args and not path): + if not func.optionalrepo: + if func.inferrepo and args and not path: # try to infer -R from command args repos = map(cmdutil.findrepo, args) guess = repos[0] @@ -883,7 +790,7 @@ if not path: raise error.RepoError(_("no repository found in" " '%s' (.hg not found)") - % os.getcwd()) + % pycompat.getcwd()) raise if repo: ui = repo.ui
--- a/mercurial/error.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/error.py Sun Nov 27 20:44:52 2016 -0500 @@ -84,7 +84,7 @@ """Raised when an update is aborted because there is nothing to merge""" class ManyMergeDestAbort(MergeDestAbort): - """Raised when an update is aborted because destination is ambigious""" + """Raised when an update is aborted because destination is ambiguous""" class ResponseExpected(Abort): """Raised when an EOF is received for a prompt"""
--- a/mercurial/exchange.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/exchange.py Sun Nov 27 20:44:52 2016 -0500 @@ -37,12 +37,6 @@ urlerr = util.urlerr urlreq = util.urlreq -# Maps bundle compression human names to internal representation. -_bundlespeccompressions = {'none': None, - 'bzip2': 'BZ', - 'gzip': 'GZ', - } - # Maps bundle version human names to changegroup versions. _bundlespeccgversions = {'v1': '01', 'v2': '02', @@ -64,7 +58,7 @@ Where <compression> is one of the supported compression formats and <type> is (currently) a version string. A ";" can follow the type and - all text afterwards is interpretted as URI encoded, ";" delimited key=value + all text afterwards is interpreted as URI encoded, ";" delimited key=value pairs. If ``strict`` is True (the default) <compression> is required. Otherwise, @@ -114,7 +108,7 @@ if '-' in spec: compression, version = spec.split('-', 1) - if compression not in _bundlespeccompressions: + if compression not in util.compengines.supportedbundlenames: raise error.UnsupportedBundleSpecification( _('%s compression is not supported') % compression) @@ -130,7 +124,7 @@ spec, params = parseparams(spec) - if spec in _bundlespeccompressions: + if spec in util.compengines.supportedbundlenames: compression = spec version = 'v1' if 'generaldelta' in repo.requirements: @@ -157,7 +151,8 @@ ', '.join(sorted(missingreqs))) if not externalnames: - compression = _bundlespeccompressions[compression] + engine = util.compengines.forbundlename(compression) + compression = engine.bundletype()[1] version = _bundlespeccgversions[version] return compression, version, params @@ -196,10 +191,10 @@ restored. """ def speccompression(alg): - for k, v in _bundlespeccompressions.items(): - if v == alg: - return k - return None + try: + return util.compengines.forbundletype(alg).bundletype()[0] + except KeyError: + return None b = readbundle(ui, fh, None) if isinstance(b, changegroup.cg1unpacker): @@ -282,7 +277,7 @@ This function is used to allow testing of the older bundle version""" ui = op.repo.ui forcebundle1 = False - # The goal is this config is to allow developper to choose the bundle + # The goal is this config is to allow developer to choose the bundle # version used during exchanged. This is especially handy during test. # Value is a list of bundle version to be picked from, highest version # should be used. @@ -1425,7 +1420,7 @@ pullop.stepsdone.add('phases') publishing = bool(remotephases.get('publishing', False)) if remotephases and not publishing: - # remote is new and unpublishing + # remote is new and non-publishing pheads, _dr = phases.analyzeremotephases(pullop.repo, pullop.pulledsubset, remotephases) @@ -1666,6 +1661,17 @@ if chunks: bundler.newpart('hgtagsfnodes', data=''.join(chunks)) +def _getbookmarks(repo, **kwargs): + """Returns bookmark to node mapping. + + This function is primarily used to generate `bookmarks` bundle2 part. + It is a separate function in order to make it easy to wrap it + in extensions. Passing `kwargs` to the function makes it easy to + add new parameters in extensions. + """ + + return dict(bookmod.listbinbookmarks(repo)) + def check_heads(repo, their_heads, context): """check if the heads of a repo have been modified
--- a/mercurial/extensions.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/extensions.py Sun Nov 27 20:44:52 2016 -0500 @@ -426,7 +426,7 @@ file.close() if doc: # extracting localized synopsis - return gettext(doc).splitlines()[0] + return gettext(doc) else: return _('(no help text available)') @@ -448,7 +448,7 @@ for name, path in paths.iteritems(): doc = _disabledhelp(path) if doc: - exts[name] = doc + exts[name] = doc.splitlines()[0] return exts
--- a/mercurial/filemerge.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/filemerge.py Sun Nov 27 20:44:52 2016 -0500 @@ -189,7 +189,8 @@ if _toolbool(ui, t, "disabled", False): disabled.add(t) names = tools.keys() - tools = sorted([(-p, t) for t, p in tools.items() if t not in disabled]) + tools = sorted([(-p, tool) for tool, p in tools.items() + if tool not in disabled]) uimerge = ui.config("ui", "merge") if uimerge: # external tools defined in uimerge won't be able to handle @@ -517,7 +518,8 @@ return util.ellipsis(mark, 80 - 8) _defaultconflictmarker = ('{node|short} ' - '{ifeq(tags, "tip", "", "{tags} ")}' + '{ifeq(tags, "tip", "", ' + 'ifeq(tags, "", "", "{tags} "))}' '{if(bookmarks, "{bookmarks} ")}' '{ifeq(branch, "default", "", "{branch} ")}' '- {author|user}: {desc|firstline}') @@ -575,8 +577,9 @@ a boolean indicating whether the file was deleted from disk.""" def temp(prefix, ctx): - pre = "%s~%s." % (os.path.basename(ctx.path()), prefix) - (fd, name) = tempfile.mkstemp(prefix=pre) + fullbase, ext = os.path.splitext(ctx.path()) + pre = "%s~%s." % (os.path.basename(fullbase), prefix) + (fd, name) = tempfile.mkstemp(prefix=pre, suffix=ext) data = repo.wwritedata(ctx.path(), ctx.data()) f = os.fdopen(fd, "wb") f.write(data)
--- a/mercurial/hbisect.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hbisect.py Sun Nov 27 20:44:52 2016 -0500 @@ -98,7 +98,7 @@ tot = len(candidates) unskipped = [c for c in candidates if (c not in skip) and (c != badrev)] if tot == 1 or not unskipped: - return ([changelog.node(rev) for rev in candidates], 0, good) + return ([changelog.node(c) for c in candidates], 0, good) perfect = tot // 2 # find the best node to test
--- a/mercurial/help/config.txt Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/help/config.txt Sun Nov 27 20:44:52 2016 -0500 @@ -1401,7 +1401,7 @@ ``type`` The type of profiler to use. - (default: ls) + (default: stat) ``ls`` Use Python's built-in instrumenting profiler. This profiler @@ -1409,9 +1409,9 @@ first line of a function. This restriction makes it difficult to identify the expensive parts of a non-trivial function. ``stat`` - Use a third-party statistical profiler, statprof. This profiler - currently runs only on Unix systems, and is most useful for - profiling commands that run for longer than about 0.1 seconds. + Use a statistical profiler, statprof. This profiler is most + useful for profiling commands that run for longer than about 0.1 + seconds. ``format`` Profiling format. Specific to the ``ls`` instrumenting profiler. @@ -1426,6 +1426,20 @@ file, the generated file can directly be loaded into kcachegrind. +``statformat`` + Profiling format for the ``stat`` profiler. + (default: hotpath) + + ``hotpath`` + Show a tree-based display containing the hot path of execution (where + most time was spent). + ``bymethod`` + Show a table of methods ordered by how frequently they are active. + ``byline`` + Show a table of lines in files ordered by how frequently they are active. + ``json`` + Render profiling data as JSON. + ``frequency`` Sampling frequency. Specific to the ``stat`` sampling profiler. (default: 1000)
--- a/mercurial/help/internals/revlogs.txt Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/help/internals/revlogs.txt Sun Nov 27 20:44:52 2016 -0500 @@ -88,11 +88,13 @@ 0-5 (6 bytes) Absolute offset of revision data from beginning of revlog. 6-7 (2 bytes) - Bit flags impacting revision behavior. + Bit flags impacting revision behavior. The following bit offsets define: + 0: 'censor' extension flag. 8-11 (4 bytes) Compressed length of revision data / chunk as stored in revlog. 12-15 (4 bytes) - Uncompressed length of revision data / chunk. + Uncompressed length of revision data. This is the size of the full + revision data, not the size of the chunk post decompression. 16-19 (4 bytes) Base or previous revision this revision's delta was produced against. -1 means this revision holds full text (as opposed to a delta).
--- a/mercurial/hgweb/__init__.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hgweb/__init__.py Sun Nov 27 20:44:52 2016 -0500 @@ -85,41 +85,11 @@ def run(self): self.httpd.serve_forever() -def createservice(ui, repo, opts): - # this way we can check if something was given in the command-line - if opts.get('port'): - opts['port'] = util.getport(opts.get('port')) - - alluis = set([ui]) - if repo: - baseui = repo.baseui - alluis.update([repo.baseui, repo.ui]) - else: - baseui = ui - webconf = opts.get('web_conf') or opts.get('webdir_conf') +def createapp(baseui, repo, webconf): if webconf: - # load server settings (e.g. web.port) to "copied" ui, which allows - # hgwebdir to reload webconf cleanly - servui = ui.copy() - servui.readconfig(webconf, sections=['web']) - alluis.add(servui) - else: - servui = ui - - optlist = ("name templates style address port prefix ipv6" - " accesslog errorlog certificate encoding") - for o in optlist.split(): - val = opts.get(o, '') - if val in (None, ''): # should check against default options instead - continue - for u in alluis: - u.setconfig("web", o, val, 'serve') - - if webconf: - app = hgwebdir_mod.hgwebdir(webconf, baseui=baseui) + return hgwebdir_mod.hgwebdir(webconf, baseui=baseui) else: if not repo: raise error.RepoError(_("there is no Mercurial repository" " here (.hg not found)")) - app = hgweb_mod.hgweb(repo, baseui=baseui) - return httpservice(servui, app, opts) + return hgweb_mod.hgweb(repo, baseui=baseui)
--- a/mercurial/hgweb/protocol.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hgweb/protocol.py Sun Nov 27 20:44:52 2016 -0500 @@ -8,7 +8,6 @@ from __future__ import absolute_import import cgi -import zlib from .common import ( HTTP_OK, @@ -74,28 +73,12 @@ self.ui.ferr, self.ui.fout = self.oldio return val - def groupchunks(self, fh): - def getchunks(): - while True: - chunk = fh.read(32768) - if not chunk: - break - yield chunk - - return self.compresschunks(getchunks()) - def compresschunks(self, chunks): # Don't allow untrusted settings because disabling compression or # setting a very high compression level could lead to flooding # the server's network or CPU. - z = zlib.compressobj(self.ui.configint('server', 'zliblevel', -1)) - for chunk in chunks: - data = z.compress(chunk) - # Not all calls to compress() emit data. It is cheaper to inspect - # that here than to send it via the generator. - if data: - yield data - yield z.flush() + opts = {'level': self.ui.configint('server', 'zliblevel', -1)} + return util.compengines['zlib'].compressstream(chunks, opts) def _client(self): return 'remote:%s:%s:%s' % ( @@ -113,8 +96,16 @@ req.respond(HTTP_OK, HGTYPE, body=rsp) return [] elif isinstance(rsp, wireproto.streamres): + if rsp.reader: + gen = iter(lambda: rsp.reader.read(32768), '') + else: + gen = rsp.gen + + if rsp.v1compressible: + gen = p.compresschunks(gen) + req.respond(HTTP_OK, HGTYPE) - return rsp.gen + return gen elif isinstance(rsp, wireproto.pushres): val = p.restore() rsp = '%d\n%s' % (rsp.res, val)
--- a/mercurial/hgweb/server.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hgweb/server.py Sun Nov 27 20:44:52 2016 -0500 @@ -281,8 +281,8 @@ prefix = '/' + prefix.strip('/') self.prefix = prefix - alog = openlog(ui.config('web', 'accesslog', '-'), sys.stdout) - elog = openlog(ui.config('web', 'errorlog', '-'), sys.stderr) + alog = openlog(ui.config('web', 'accesslog', '-'), ui.fout) + elog = openlog(ui.config('web', 'errorlog', '-'), ui.ferr) self.accesslog = alog self.errorlog = elog
--- a/mercurial/hgweb/webutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hgweb/webutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -269,7 +269,7 @@ try: ctx = repo[changeid] except error.RepoError: - man = repo.manifest + man = repo.manifestlog._revlog ctx = repo[man.linkrev(man.rev(man.lookup(changeid)))] return ctx
--- a/mercurial/hgweb/wsgicgi.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hgweb/wsgicgi.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,7 +11,6 @@ from __future__ import absolute_import import os -import sys from .. import ( util, @@ -22,8 +21,8 @@ ) def launch(application): - util.setbinary(sys.stdin) - util.setbinary(sys.stdout) + util.setbinary(util.stdin) + util.setbinary(util.stdout) environ = dict(os.environ.iteritems()) environ.setdefault('PATH_INFO', '') @@ -33,12 +32,12 @@ if environ['PATH_INFO'].startswith(scriptname): environ['PATH_INFO'] = environ['PATH_INFO'][len(scriptname):] - stdin = sys.stdin + stdin = util.stdin if environ.get('HTTP_EXPECT', '').lower() == '100-continue': - stdin = common.continuereader(stdin, sys.stdout.write) + stdin = common.continuereader(stdin, util.stdout.write) environ['wsgi.input'] = stdin - environ['wsgi.errors'] = sys.stderr + environ['wsgi.errors'] = util.stderr environ['wsgi.version'] = (1, 0) environ['wsgi.multithread'] = False environ['wsgi.multiprocess'] = True @@ -51,7 +50,7 @@ headers_set = [] headers_sent = [] - out = sys.stdout + out = util.stdout def write(data): if not headers_set:
--- a/mercurial/hook.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/hook.py Sun Nov 27 20:44:52 2016 -0500 @@ -16,6 +16,7 @@ demandimport, error, extensions, + pycompat, util, ) @@ -90,12 +91,6 @@ starttime = time.time() try: - # redirect IO descriptors to the ui descriptors so hooks - # that write directly to these don't mess up the command - # protocol when running through the command server - old = sys.stdout, sys.stderr, sys.stdin - sys.stdout, sys.stderr, sys.stdin = ui.fout, ui.ferr, ui.fin - r = obj(ui=ui, repo=repo, hooktype=name, **args) except Exception as exc: if isinstance(exc, error.Abort): @@ -111,7 +106,6 @@ ui.traceback() return True, True finally: - sys.stdout, sys.stderr, sys.stdin = old duration = time.time() - starttime ui.log('pythonhook', 'pythonhook-%s: %s finished in %0.2f seconds\n', name, funcname, duration) @@ -148,7 +142,7 @@ if repo: cwd = repo.root else: - cwd = os.getcwd() + cwd = pycompat.getcwd() r = ui.system(cmd, environ=env, cwd=cwd) duration = time.time() - starttime @@ -216,11 +210,11 @@ for hname, cmd in hooks: if oldstdout == -1 and _redirect: try: - stdoutno = sys.__stdout__.fileno() - stderrno = sys.__stderr__.fileno() + stdoutno = util.stdout.fileno() + stderrno = util.stderr.fileno() # temporarily redirect stdout to stderr, if possible if stdoutno >= 0 and stderrno >= 0: - sys.__stdout__.flush() + util.stdout.flush() oldstdout = os.dup(stdoutno) os.dup2(stderrno, stdoutno) except (OSError, AttributeError): @@ -262,9 +256,10 @@ # The stderr is fully buffered on Windows when connected to a pipe. # A forcible flush is required to make small stderr data in the # remote side available to the client immediately. - sys.stderr.flush() + util.stderr.flush() finally: if _redirect and oldstdout >= 0: + util.stdout.flush() # write hook output to stderr fd os.dup2(oldstdout, stdoutno) os.close(oldstdout)
--- a/mercurial/httppeer.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/httppeer.py Sun Nov 27 20:44:52 2016 -0500 @@ -12,7 +12,6 @@ import os import socket import tempfile -import zlib from .i18n import _ from .node import nullid @@ -30,16 +29,29 @@ urlerr = util.urlerr urlreq = util.urlreq -def zgenerator(f): - zd = zlib.decompressobj() +# FUTURE: consider refactoring this API to use generators. This will +# require a compression engine API to emit generators. +def decompressresponse(response, engine): try: - for chunk in util.filechunkiter(f): - while chunk: - yield zd.decompress(chunk, 2**18) - chunk = zd.unconsumed_tail + reader = engine.decompressorreader(response) except httplib.HTTPException: raise IOError(None, _('connection ended unexpectedly')) - yield zd.flush() + + # We need to wrap reader.read() so HTTPException on subsequent + # reads is also converted. + # Ideally we'd use super() here. However, if ``reader`` isn't a new-style + # class, this can raise: + # TypeError: super() argument 1 must be type, not classobj + origread = reader.read + class readerproxy(reader.__class__): + def read(self, *args, **kwargs): + try: + return origread(*args, **kwargs) + except httplib.HTTPException: + raise IOError(None, _('connection ended unexpectedly')) + + reader.__class__ = readerproxy + return reader class httppeer(wireproto.wirepeer): def __init__(self, ui, path): @@ -90,7 +102,7 @@ def lock(self): raise error.Abort(_('operation not supported over http')) - def _callstream(self, cmd, **args): + def _callstream(self, cmd, _compressible=False, **args): if cmd == 'pushkey': args['data'] = '' data = args.pop('data', None) @@ -160,9 +172,6 @@ self.ui.debug('http error while sending %s command\n' % cmd) self.ui.traceback() raise IOError(None, inst) - except IndexError: - # this only happens with Python 2.3, later versions raise URLError - raise error.Abort(_('http error, possibly caused by proxy setting')) # record the url we got redirected to resp_url = resp.geturl() if resp_url.endswith(qs): @@ -201,6 +210,9 @@ raise error.RepoError(_("'%s' uses newer protocol %s") % (safeurl, version)) + if _compressible: + return decompressresponse(resp, util.compengines['zlib']) + return resp def _call(self, cmd, **args): @@ -271,8 +283,7 @@ os.unlink(filename) def _callcompressable(self, cmd, **args): - stream = self._callstream(cmd, **args) - return util.chunkbuffer(zgenerator(stream)) + return self._callstream(cmd, _compressible=True, **args) def _abort(self, exception): raise exception
--- a/mercurial/i18n.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/i18n.py Sun Nov 27 20:44:52 2016 -0500 @@ -49,6 +49,7 @@ _ugettext = None def setdatapath(datapath): + datapath = pycompat.fsdecode(datapath) localedir = os.path.join(datapath, pycompat.sysstr('locale')) t = gettextmod.translation('hg', localedir, _languages, fallback=True) global _ugettext
--- a/mercurial/keepalive.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/keepalive.py Sun Nov 27 20:44:52 2016 -0500 @@ -78,31 +78,6 @@ easy to distinguish between non-200 responses. The reason is that urllib2 tries to do clever things with error codes 301, 302, 401, and 407, and it wraps the object upon return. - - For python versions earlier than 2.4, you can avoid this fancy error - handling by setting the module-level global HANDLE_ERRORS to zero. - You see, prior to 2.4, it's the HTTP Handler's job to determine what - to handle specially, and what to just pass up. HANDLE_ERRORS == 0 - means "pass everything up". In python 2.4, however, this job no - longer belongs to the HTTP Handler and is now done by a NEW handler, - HTTPErrorProcessor. Here's the bottom line: - - python version < 2.4 - HANDLE_ERRORS == 1 (default) pass up 200, treat the rest as - errors - HANDLE_ERRORS == 0 pass everything up, error processing is - left to the calling code - python version >= 2.4 - HANDLE_ERRORS == 1 pass up 200, treat the rest as errors - HANDLE_ERRORS == 0 (default) pass everything up, let the - other handlers (specifically, - HTTPErrorProcessor) decide what to do - - In practice, setting the variable either way makes little difference - in python 2.4, so for the most consistent behavior across versions, - you probably just want to use the defaults, which will give you - exceptions on errors. - """ # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $ @@ -125,10 +100,6 @@ DEBUG = None -if sys.version_info < (2, 4): - HANDLE_ERRORS = 1 -else: HANDLE_ERRORS = 0 - class ConnectionManager(object): """ The connection manager must be able to: @@ -277,11 +248,7 @@ r.headers = r.msg r.msg = r.reason - if r.status == 200 or not HANDLE_ERRORS: - return r - else: - return self.parent.error('http', req, r, - r.status, r.msg, r.headers) + return r def _reuse_connection(self, h, req, host): """start the transaction with a re-used connection @@ -332,10 +299,9 @@ def _start_transaction(self, h, req): # What follows mostly reimplements HTTPConnection.request() # except it adds self.parent.addheaders in the mix. - headers = req.headers.copy() - if sys.version_info >= (2, 4): - headers.update(req.unredirected_hdrs) - headers.update(self.parent.addheaders) + headers = dict(self.parent.addheaders) + headers.update(req.headers) + headers.update(req.unredirected_hdrs) headers = dict((n.lower(), v) for n, v in headers.items()) skipheaders = {} for n in ('host', 'accept-encoding'): @@ -596,33 +562,6 @@ ##### TEST FUNCTIONS ######################################################################### -def error_handler(url): - global HANDLE_ERRORS - orig = HANDLE_ERRORS - keepalive_handler = HTTPHandler() - opener = urlreq.buildopener(keepalive_handler) - urlreq.installopener(opener) - pos = {0: 'off', 1: 'on'} - for i in (0, 1): - print(" fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i)) - HANDLE_ERRORS = i - try: - fo = urlreq.urlopen(url) - fo.read() - fo.close() - try: - status, reason = fo.status, fo.reason - except AttributeError: - status, reason = None, None - except IOError as e: - print(" EXCEPTION: %s" % e) - raise - else: - print(" status = %s, reason = %s" % (status, reason)) - HANDLE_ERRORS = orig - hosts = keepalive_handler.open_connections() - print("open connections:", hosts) - keepalive_handler.close_all() def continuity(url): md5 = hashlib.md5 @@ -661,14 +600,14 @@ def comp(N, url): print(' making %i connections to:\n %s' % (N, url)) - sys.stdout.write(' first using the normal urllib handlers') + util.stdout.write(' first using the normal urllib handlers') # first use normal opener opener = urlreq.buildopener() urlreq.installopener(opener) t1 = fetch(N, url) print(' TIME: %.3f s' % t1) - sys.stdout.write(' now using the keepalive handler ') + util.stdout.write(' now using the keepalive handler ') # now install the keepalive handler and try again opener = urlreq.buildopener(HTTPHandler()) urlreq.installopener(opener) @@ -713,11 +652,11 @@ i = 20 print(" waiting %i seconds for the server to close the connection" % i) while i > 0: - sys.stdout.write('\r %2i' % i) - sys.stdout.flush() + util.stdout.write('\r %2i' % i) + util.stdout.flush() time.sleep(1) i -= 1 - sys.stderr.write('\r') + util.stderr.write('\r') print(" fetching the file a second time") fo = urlreq.urlopen(url) @@ -733,12 +672,6 @@ def test(url, N=10): - print("checking error handler (do this on a non-200)") - try: error_handler(url) - except IOError: - print("exiting - exception will prevent further tests") - sys.exit() - print('') print("performing continuity test (making sure stuff isn't corrupted)") continuity(url) print('')
--- a/mercurial/localrepo.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/localrepo.py Sun Nov 27 20:44:52 2016 -0500 @@ -28,9 +28,9 @@ bundle2, changegroup, changelog, - cmdutil, context, dirstate, + dirstateguard, encoding, error, exchange, @@ -41,6 +41,7 @@ manifest, match as matchmod, merge as mergemod, + mergeutil, namespaces, obsolete, pathutil, @@ -504,15 +505,11 @@ c.readpending('00changelog.i.a') return c - @property - def manifest(self): - return self.manifestlog._oldmanifest - def _constructmanifest(self): # This is a temporary function while we migrate from manifest to # manifestlog. It allows bundlerepo and unionrepo to intercept the # manifest creation. - return manifest.manifest(self.svfs) + return manifest.manifestrevlog(self.svfs) @storecache('00manifest.i') def manifestlog(self): @@ -1145,7 +1142,7 @@ wlock = self.wlock() lock = self.lock() if self.svfs.exists("undo"): - dsguard = cmdutil.dirstateguard(self, 'rollback') + dsguard = dirstateguard.dirstateguard(self, 'rollback') return self._rollback(dryrun, force, dsguard) else: @@ -1502,7 +1499,7 @@ return fparent1 def checkcommitpatterns(self, wctx, vdirs, match, status, fail): - """check for commit arguments that aren't commitable""" + """check for commit arguments that aren't committable""" if match.isexact() or match.prefix(): matched = set(status.modified + status.added + status.removed) @@ -1633,13 +1630,7 @@ raise error.Abort(_("cannot commit merge with missing files")) ms = mergemod.mergestate.read(self) - - if list(ms.unresolved()): - raise error.Abort(_("unresolved merge conflicts " - "(see 'hg help resolve')")) - if ms.mdstate() != 's' or list(ms.driverresolved()): - raise error.Abort(_('driver-resolved merge conflicts'), - hint=_('run "hg resolve --all" to resolve')) + mergeutil.checkunresolved(ms) if editor: cctx._text = editor(self, cctx, subs) @@ -1706,9 +1697,13 @@ trp = weakref.proxy(tr) if ctx.files(): - m1 = p1.manifest() - m2 = p2.manifest() - m = m1.copy() + m1ctx = p1.manifestctx() + m2ctx = p2.manifestctx() + mctx = m1ctx.copy() + + m = mctx.read() + m1 = m1ctx.read() + m2 = m2ctx.read() # check in files added = [] @@ -1742,9 +1737,9 @@ drop = [f for f in removed if f in m] for f in drop: del m[f] - mn = self.manifestlog.add(m, trp, linkrev, - p1.manifestnode(), p2.manifestnode(), - added, drop) + mn = mctx.write(trp, linkrev, + p1.manifestnode(), p2.manifestnode(), + added, drop) files = changed + removed else: mn = p1.manifestnode()
--- a/mercurial/mail.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/mail.py Sun Nov 27 20:44:52 2016 -0500 @@ -5,7 +5,7 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import email import email.charset @@ -14,7 +14,6 @@ import quopri import smtplib import socket -import sys import time from .i18n import _ @@ -87,7 +86,7 @@ def _get_socket(self, host, port, timeout): if self.debuglevel > 0: - print('connect:', (host, port), file=sys.stderr) + self._ui.debug('connect: %r\n' % (host, port)) new_socket = socket.create_connection((host, port), timeout) new_socket = sslutil.wrapsocket(new_socket, self.keyfile, self.certfile,
--- a/mercurial/manifest.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/manifest.py Sun Nov 27 20:44:52 2016 -0500 @@ -422,6 +422,11 @@ def __len__(self): return len(self._lm) + def __nonzero__(self): + # nonzero is covered by the __len__ function, but implementing it here + # makes it easier for extensions to override. + return len(self._lm) != 0 + def __setitem__(self, key, node): self._lm[key] = node, self.flags(key, '') @@ -1178,7 +1183,7 @@ self._dirlogcache) return self._dirlogcache[dir] - def add(self, m, transaction, link, p1, p2, added, removed): + def add(self, m, transaction, link, p1, p2, added, removed, readtree=None): if (p1 in self.fulltextcache and util.safehasattr(m, 'fastdelta') and not self._usemanifestv2): # If our first parent is in the manifest cache, we can @@ -1201,9 +1206,10 @@ # through to the revlog layer, and let it handle the delta # process. if self._treeondisk: - m1 = self.read(p1) - m2 = self.read(p2) - n = self._addtree(m, transaction, link, m1, m2) + assert readtree, "readtree must be set for treemanifest writes" + m1 = readtree(self._dir, p1) + m2 = readtree(self._dir, p2) + n = self._addtree(m, transaction, link, m1, m2, readtree) arraytext = None else: text = m.text(self._usemanifestv2) @@ -1215,14 +1221,15 @@ return n - def _addtree(self, m, transaction, link, m1, m2): + def _addtree(self, m, transaction, link, m1, m2, readtree): # If the manifest is unchanged compared to one parent, # don't write a new revision if m.unmodifiedsince(m1) or m.unmodifiedsince(m2): return m.node() def writesubtree(subm, subp1, subp2): sublog = self.dirlog(subm.dir()) - sublog.add(subm, transaction, link, subp1, subp2, None, None) + sublog.add(subm, transaction, link, subp1, subp2, None, None, + readtree=readtree) m.writesubtrees(m1, m2, writesubtree) text = m.dirtext(self._usemanifestv2) # Double-check whether contents are unchanged to one parent @@ -1248,41 +1255,100 @@ self._repo = repo usetreemanifest = False + cachesize = 4 opts = getattr(opener, 'options', None) if opts is not None: usetreemanifest = opts.get('treemanifest', usetreemanifest) + cachesize = opts.get('manifestcachesize', cachesize) self._treeinmem = usetreemanifest self._oldmanifest = repo._constructmanifest() self._revlog = self._oldmanifest - # We'll separate this into it's own cache once oldmanifest is no longer - # used - self._mancache = self._oldmanifest._mancache + # A cache of the manifestctx or treemanifestctx for each directory + self._dirmancache = {} + self._dirmancache[''] = util.lrucachedict(cachesize) + + self.cachesize = cachesize def __getitem__(self, node): - """Retrieves the manifest instance for the given node. Throws a KeyError - if not found. + """Retrieves the manifest instance for the given node. Throws a + LookupError if not found. """ - if node in self._mancache: - cachemf = self._mancache[node] - # The old manifest may put non-ctx manifests in the cache, so skip - # those since they don't implement the full api. + return self.get('', node) + + def get(self, dir, node, verify=True): + """Retrieves the manifest instance for the given node. Throws a + LookupError if not found. + + `verify` - if True an exception will be thrown if the node is not in + the revlog + """ + if node in self._dirmancache.get(dir, ()): + cachemf = self._dirmancache[dir][node] + # The old manifest may put non-ctx manifests in the cache, so + # skip those since they don't implement the full api. if (isinstance(cachemf, manifestctx) or isinstance(cachemf, treemanifestctx)): return cachemf - if self._treeinmem: - m = treemanifestctx(self._repo, '', node) + if dir: + if self._revlog._treeondisk: + if verify: + dirlog = self._revlog.dirlog(dir) + if node not in dirlog.nodemap: + raise LookupError(node, dirlog.indexfile, + _('no node')) + m = treemanifestctx(self._repo, dir, node) + else: + raise error.Abort( + _("cannot ask for manifest directory '%s' in a flat " + "manifest") % dir) else: - m = manifestctx(self._repo, node) + if verify: + if node not in self._revlog.nodemap: + raise LookupError(node, self._revlog.indexfile, + _('no node')) + if self._treeinmem: + m = treemanifestctx(self._repo, '', node) + else: + m = manifestctx(self._repo, node) + if node != revlog.nullid: - self._mancache[node] = m + mancache = self._dirmancache.get(dir) + if not mancache: + mancache = util.lrucachedict(self.cachesize) + self._dirmancache[dir] = mancache + mancache[node] = m return m - def add(self, m, transaction, link, p1, p2, added, removed): - return self._revlog.add(m, transaction, link, p1, p2, added, removed) + def clearcaches(self): + self._dirmancache.clear() + self._revlog.clearcaches() + +class memmanifestctx(object): + def __init__(self, repo): + self._repo = repo + self._manifestdict = manifestdict() + + def _revlog(self): + return self._repo.manifestlog._revlog + + def new(self): + return memmanifestctx(self._repo) + + def copy(self): + memmf = memmanifestctx(self._repo) + memmf._manifestdict = self.read().copy() + return memmf + + def read(self): + return self._manifestdict + + def write(self, transaction, link, p1, p2, added, removed): + return self._revlog().add(self._manifestdict, transaction, link, p1, p2, + added, removed) class manifestctx(object): """A class representing a single revision of a manifest, including its @@ -1301,35 +1367,58 @@ #rev = revlog.rev(node) #self.linkrev = revlog.linkrev(rev) + def _revlog(self): + return self._repo.manifestlog._revlog + def node(self): return self._node + def new(self): + return memmanifestctx(self._repo) + + def copy(self): + memmf = memmanifestctx(self._repo) + memmf._manifestdict = self.read().copy() + return memmf + def read(self): if not self._data: if self._node == revlog.nullid: self._data = manifestdict() else: - rl = self._repo.manifestlog._revlog + rl = self._revlog() text = rl.revision(self._node) arraytext = array.array('c', text) rl._fulltextcache[self._node] = arraytext self._data = manifestdict(text) return self._data - def readfast(self): - rl = self._repo.manifestlog._revlog + def readfast(self, shallow=False): + '''Calls either readdelta or read, based on which would be less work. + readdelta is called if the delta is against the p1, and therefore can be + read quickly. + + If `shallow` is True, nothing changes since this is a flat manifest. + ''' + rl = self._revlog() r = rl.rev(self._node) deltaparent = rl.deltaparent(r) if deltaparent != revlog.nullrev and deltaparent in rl.parentrevs(r): return self.readdelta() return self.read() - def readdelta(self): - revlog = self._repo.manifestlog._revlog + def readdelta(self, shallow=False): + '''Returns a manifest containing just the entries that are present + in this manifest, but not in its p1 manifest. This is efficient to read + if the revlog delta is already p1. + + Changing the value of `shallow` has no effect on flat manifests. + ''' + revlog = self._revlog() if revlog._usemanifestv2: # Need to perform a slow delta r0 = revlog.deltaparent(revlog.rev(self._node)) - m0 = manifestctx(self._repo, revlog.node(r0)).read() + m0 = self._repo.manifestlog[revlog.node(r0)].read() m1 = self.read() md = manifestdict() for f, ((n0, fl0), (n1, fl1)) in m0.diff(m1).iteritems(): @@ -1343,6 +1432,35 @@ d = mdiff.patchtext(revlog.revdiff(revlog.deltaparent(r), r)) return manifestdict(d) + def find(self, key): + return self.read().find(key) + +class memtreemanifestctx(object): + def __init__(self, repo, dir=''): + self._repo = repo + self._dir = dir + self._treemanifest = treemanifest() + + def _revlog(self): + return self._repo.manifestlog._revlog + + def new(self, dir=''): + return memtreemanifestctx(self._repo, dir=dir) + + def copy(self): + memmf = memtreemanifestctx(self._repo, dir=self._dir) + memmf._treemanifest = self._treemanifest.copy() + return memmf + + def read(self): + return self._treemanifest + + def write(self, transaction, link, p1, p2, added, removed): + def readtree(dir, node): + return self._repo.manifestlog.get(dir, node).read() + return self._revlog().add(self._treemanifest, transaction, link, p1, p2, + added, removed, readtree=readtree) + class treemanifestctx(object): def __init__(self, repo, dir, node): self._repo = repo @@ -1371,12 +1489,15 @@ def gettext(): return rl.revision(self._node) def readsubtree(dir, subm): - return treemanifestctx(self._repo, dir, subm).read() + # Set verify to False since we need to be able to create + # subtrees for trees that don't exist on disk. + return self._repo.manifestlog.get(dir, subm, + verify=False).read() m.read(gettext, readsubtree) m.setnode(self._node) self._data = m else: - text = revlog.revision(self._node) + text = rl.revision(self._node) arraytext = array.array('c', text) rl.fulltextcache[self._node] = arraytext self._data = treemanifest(dir=self._dir, text=text) @@ -1386,150 +1507,62 @@ def node(self): return self._node - def readdelta(self): - # Need to perform a slow delta + def new(self, dir=''): + return memtreemanifestctx(self._repo, dir=dir) + + def copy(self): + memmf = memtreemanifestctx(self._repo, dir=self._dir) + memmf._treemanifest = self.read().copy() + return memmf + + def readdelta(self, shallow=False): + '''Returns a manifest containing just the entries that are present + in this manifest, but not in its p1 manifest. This is efficient to read + if the revlog delta is already p1. + + If `shallow` is True, this will read the delta for this directory, + without recursively reading subdirectory manifests. Instead, any + subdirectory entry will be reported as it appears in the manifest, i.e. + the subdirectory will be reported among files and distinguished only by + its 't' flag. + ''' revlog = self._revlog() - r0 = revlog.deltaparent(revlog.rev(self._node)) - m0 = treemanifestctx(self._repo, self._dir, revlog.node(r0)).read() - m1 = self.read() - md = treemanifest(dir=self._dir) - for f, ((n0, fl0), (n1, fl1)) in m0.diff(m1).iteritems(): - if n1: - md[f] = n1 - if fl1: - md.setflag(f, fl1) - return md + if shallow and not revlog._usemanifestv2: + r = revlog.rev(self._node) + d = mdiff.patchtext(revlog.revdiff(revlog.deltaparent(r), r)) + return manifestdict(d) + else: + # Need to perform a slow delta + r0 = revlog.deltaparent(revlog.rev(self._node)) + m0 = self._repo.manifestlog.get(self._dir, revlog.node(r0)).read() + m1 = self.read() + md = treemanifest(dir=self._dir) + for f, ((n0, fl0), (n1, fl1)) in m0.diff(m1).iteritems(): + if n1: + md[f] = n1 + if fl1: + md.setflag(f, fl1) + return md - def readfast(self): + def readfast(self, shallow=False): + '''Calls either readdelta or read, based on which would be less work. + readdelta is called if the delta is against the p1, and therefore can be + read quickly. + + If `shallow` is True, it only returns the entries from this manifest, + and not any submanifests. + ''' rl = self._revlog() r = rl.rev(self._node) deltaparent = rl.deltaparent(r) - if deltaparent != revlog.nullrev and deltaparent in rl.parentrevs(r): - return self.readdelta() - return self.read() - -class manifest(manifestrevlog): - def __init__(self, opener, dir='', dirlogcache=None): - '''The 'dir' and 'dirlogcache' arguments are for internal use by - manifest.manifest only. External users should create a root manifest - log with manifest.manifest(opener) and call dirlog() on it. - ''' - # During normal operations, we expect to deal with not more than four - # revs at a time (such as during commit --amend). When rebasing large - # stacks of commits, the number can go up, hence the config knob below. - cachesize = 4 - usetreemanifest = False - opts = getattr(opener, 'options', None) - if opts is not None: - cachesize = opts.get('manifestcachesize', cachesize) - usetreemanifest = opts.get('treemanifest', usetreemanifest) - self._mancache = util.lrucachedict(cachesize) - self._treeinmem = usetreemanifest - super(manifest, self).__init__(opener, dir=dir, dirlogcache=dirlogcache) - - def _newmanifest(self, data=''): - if self._treeinmem: - return treemanifest(self._dir, data) - return manifestdict(data) - - def dirlog(self, dir): - """This overrides the base revlog implementation to allow construction - 'manifest' types instead of manifestrevlog types. This is only needed - until we migrate off the 'manifest' type.""" - if dir: - assert self._treeondisk - if dir not in self._dirlogcache: - self._dirlogcache[dir] = manifest(self.opener, dir, - self._dirlogcache) - return self._dirlogcache[dir] - - def _slowreaddelta(self, node): - r0 = self.deltaparent(self.rev(node)) - m0 = self.read(self.node(r0)) - m1 = self.read(node) - md = self._newmanifest() - for f, ((n0, fl0), (n1, fl1)) in m0.diff(m1).iteritems(): - if n1: - md[f] = n1 - if fl1: - md.setflag(f, fl1) - return md - - def readdelta(self, node): - if self._usemanifestv2 or self._treeondisk: - return self._slowreaddelta(node) - r = self.rev(node) - d = mdiff.patchtext(self.revdiff(self.deltaparent(r), r)) - return self._newmanifest(d) + if (deltaparent != revlog.nullrev and + deltaparent in rl.parentrevs(r)): + return self.readdelta(shallow=shallow) - def readshallowdelta(self, node): - '''For flat manifests, this is the same as readdelta(). For - treemanifests, this will read the delta for this revlog's directory, - without recursively reading subdirectory manifests. Instead, any - subdirectory entry will be reported as it appears in the manifests, i.e. - the subdirectory will be reported among files and distinguished only by - its 't' flag.''' - if not self._treeondisk: - return self.readdelta(node) - if self._usemanifestv2: - raise error.Abort( - _("readshallowdelta() not implemented for manifestv2")) - r = self.rev(node) - d = mdiff.patchtext(self.revdiff(self.deltaparent(r), r)) - return manifestdict(d) - - def readshallowfast(self, node): - '''like readfast(), but calls readshallowdelta() instead of readdelta() - ''' - r = self.rev(node) - deltaparent = self.deltaparent(r) - if deltaparent != revlog.nullrev and deltaparent in self.parentrevs(r): - return self.readshallowdelta(node) - return self.readshallow(node) + if shallow: + return manifestdict(rl.revision(self._node)) + else: + return self.read() - def read(self, node): - if node == revlog.nullid: - return self._newmanifest() # don't upset local cache - if node in self._mancache: - cached = self._mancache[node] - if (isinstance(cached, manifestctx) or - isinstance(cached, treemanifestctx)): - cached = cached.read() - return cached - if self._treeondisk: - def gettext(): - return self.revision(node) - def readsubtree(dir, subm): - return self.dirlog(dir).read(subm) - m = self._newmanifest() - m.read(gettext, readsubtree) - m.setnode(node) - arraytext = None - else: - text = self.revision(node) - m = self._newmanifest(text) - arraytext = array.array('c', text) - self._mancache[node] = m - if arraytext is not None: - self.fulltextcache[node] = arraytext - return m - - def readshallow(self, node): - '''Reads the manifest in this directory. When using flat manifests, - this manifest will generally have files in subdirectories in it. Does - not cache the manifest as the callers generally do not read the same - version twice.''' - return manifestdict(self.revision(node)) - - def find(self, node, f): - '''look up entry for a single file efficiently. - return (node, flags) pair if found, (None, None) if not.''' - m = self.read(node) - try: - return m.find(f) - except KeyError: - return None, None - - def clearcaches(self): - super(manifest, self).clearcaches() - self._mancache.clear() + def find(self, key): + return self.read().find(key)
--- a/mercurial/match.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/match.py Sun Nov 27 20:44:52 2016 -0500 @@ -669,7 +669,7 @@ patterns = [] fp = open(filepath) - for lineno, line in enumerate(fp, start=1): + for lineno, line in enumerate(util.iterfile(fp), start=1): if "#" in line: global _commentre if not _commentre:
--- a/mercurial/merge.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/merge.py Sun Nov 27 20:44:52 2016 -0500 @@ -15,18 +15,20 @@ from .i18n import _ from .node import ( + addednodeid, bin, hex, + modifiednodeid, nullhex, nullid, nullrev, ) from . import ( copies, - destutil, error, filemerge, obsolete, + pycompat, scmutil, subrepo, util, @@ -66,7 +68,7 @@ (experimental) m: the external merge driver defined for this merge plus its run state (experimental) - f: a (filename, dictonary) tuple of optional values for a given file + f: a (filename, dictionary) tuple of optional values for a given file X: unsupported mandatory record type (used in tests) x: unsupported advisory record type (used in tests) l: the labels for the parts of the merge. @@ -814,7 +816,7 @@ if '.hgsubstate' in m1: # check whether sub state is modified if any(wctx.sub(s).dirty() for s in wctx.substate): - m1['.hgsubstate'] += '+' + m1['.hgsubstate'] = modifiednodeid # Compare manifests if matcher is not None: @@ -873,7 +875,7 @@ else: actions[f] = ('cd', (f, None, f, False, pa.node()), "prompt changed/deleted") - elif n1[20:] == 'a': + elif n1 == addednodeid: # This extra 'a' is added by working copy manifest to mark # the file as locally added. We should forget it instead of # deleting it. @@ -1039,7 +1041,7 @@ wjoin = repo.wjoin audit = repo.wvfs.audit try: - cwd = os.getcwd() + cwd = pycompat.getcwd() except OSError as err: if err.errno != errno.ENOENT: raise @@ -1065,7 +1067,7 @@ # cwd was present before we started to remove files # let's check if it is present after we removed them try: - os.getcwd() + pycompat.getcwd() except OSError as err: if err.errno != errno.ENOENT: raise @@ -1482,11 +1484,6 @@ if ancestor is not None: pas = [repo[ancestor]] - if node is None: - repo.ui.deprecwarn('update with no target', '3.9') - rev, _mark, _act = destutil.destupdate(repo) - node = repo[rev].node() - overwrite = force and not branchmerge p2 = repo[node]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/mergeutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,22 @@ +# mergeutil.py - help for merge processing in mercurial +# +# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from .i18n import _ + +from . import ( + error, +) + +def checkunresolved(ms): + if list(ms.unresolved()): + raise error.Abort(_("unresolved merge conflicts " + "(see 'hg help resolve')")) + if ms.mdstate() != 's' or list(ms.driverresolved()): + raise error.Abort(_('driver-resolved merge conflicts'), + hint=_('run "hg resolve --all" to resolve'))
--- a/mercurial/minirst.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/minirst.py Sun Nov 27 20:44:52 2016 -0500 @@ -697,7 +697,7 @@ if collapse: synthetic.reverse() for s in synthetic: - path = [blocks[i]['lines'][0] for i in s] + path = [blocks[syn]['lines'][0] for syn in s] real = s[-1] + 2 realline = blocks[real]['lines'] realline[0] = ('"%s"' %
--- a/mercurial/node.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/node.py Sun Nov 27 20:44:52 2016 -0500 @@ -17,6 +17,14 @@ nullid = b"\0" * 20 nullhex = hex(nullid) +# Phony node value to stand-in for new files in some uses of +# manifests. +newnodeid = '!' * 20 +addednodeid = ('0' * 15) + 'added' +modifiednodeid = ('0' * 12) + 'modified' + +wdirnodes = set((newnodeid, addednodeid, modifiednodeid)) + # pseudo identifiers for working directory # (they are experimental, so don't add too many dependencies on them) wdirrev = 0x7fffffff
--- a/mercurial/osutil.c Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/osutil.c Sun Nov 27 20:44:52 2016 -0500 @@ -727,6 +727,63 @@ } #endif /* CMSG_LEN */ + +#if defined(HAVE_SETPROCTITLE) +/* setproctitle is the first choice - available in FreeBSD */ +#define SETPROCNAME_USE_SETPROCTITLE +#elif (defined(__linux__) || defined(__APPLE__)) && PY_MAJOR_VERSION == 2 +/* rewrite the argv buffer in place - works in Linux and OS X. Py_GetArgcArgv + * in Python 3 returns the copied wchar_t **argv, thus unsupported. */ +#define SETPROCNAME_USE_ARGVREWRITE +#else +#define SETPROCNAME_USE_NONE +#endif + +#ifndef SETPROCNAME_USE_NONE +static PyObject *setprocname(PyObject *self, PyObject *args) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "s", &name)) + return NULL; + +#if defined(SETPROCNAME_USE_SETPROCTITLE) + setproctitle("%s", name); +#elif defined(SETPROCNAME_USE_ARGVREWRITE) + { + static char *argvstart = NULL; + static size_t argvsize = 0; + if (argvstart == NULL) { + int argc = 0, i; + char **argv = NULL; + char *argvend; + extern void Py_GetArgcArgv(int *argc, char ***argv); + Py_GetArgcArgv(&argc, &argv); + + /* Check the memory we can use. Typically, argv[i] and + * argv[i + 1] are continuous. */ + argvend = argvstart = argv[0]; + for (i = 0; i < argc; ++i) { + if (argv[i] > argvend || argv[i] < argvstart) + break; /* not continuous */ + size_t len = strlen(argv[i]); + argvend = argv[i] + len + 1 /* '\0' */; + } + if (argvend > argvstart) /* sanity check */ + argvsize = argvend - argvstart; + } + + if (argvstart && argvsize > 1) { + int n = snprintf(argvstart, argvsize, "%s", name); + if (n >= 0 && (size_t)n < argvsize) + memset(argvstart + n, 0, argvsize - n); + } + } +#endif + + Py_RETURN_NONE; +} +#endif /* ndef SETPROCNAME_USE_NONE */ + #endif /* ndef _WIN32 */ static PyObject *listdir(PyObject *self, PyObject *args, PyObject *kwargs) @@ -899,7 +956,11 @@ {"recvfds", (PyCFunction)recvfds, METH_VARARGS, "receive list of file descriptors via socket\n"}, #endif +#ifndef SETPROCNAME_USE_NONE + {"setprocname", (PyCFunction)setprocname, METH_VARARGS, + "set process title (best-effort)\n"}, #endif +#endif /* ndef _WIN32 */ #ifdef __APPLE__ { "isgui", (PyCFunction)isgui, METH_NOARGS,
--- a/mercurial/parser.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/parser.py Sun Nov 27 20:44:52 2016 -0500 @@ -248,7 +248,7 @@ This is a helper for fileset/revset/template aliases. A concrete rule set should be made by sub-classing this and implementing class/static methods. - It supports alias expansion of symbol and funciton-call styles:: + It supports alias expansion of symbol and function-call styles:: # decl = defn h = heads(default)
--- a/mercurial/patch.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/patch.py Sun Nov 27 20:44:52 2016 -0500 @@ -1069,7 +1069,7 @@ # Remove comment lines patchfp = open(patchfn) ncpatchfp = stringio() - for line in patchfp: + for line in util.iterfile(patchfp): if not line.startswith('#'): ncpatchfp.write(line) patchfp.close() @@ -2012,7 +2012,7 @@ fp = util.popen('%s %s -p%d < %s' % (patcher, ' '.join(args), strip, util.shellquote(patchname))) try: - for line in fp: + for line in util.iterfile(fp): line = line.rstrip() ui.note(line + '\n') if line.startswith('patching file '): @@ -2550,7 +2550,7 @@ addresult() return results -def diffstat(lines, width=80, git=False): +def diffstat(lines, width=80): output = [] stats = diffstatdata(lines) maxname, maxtotal, totaladds, totalremoves, hasbinary = diffstatsum(stats)
--- a/mercurial/pathutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/pathutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -84,7 +84,7 @@ normparts.pop() prefixes = [] # It's important that we check the path parts starting from the root. - # This means we won't accidentaly traverse a symlink into some other + # This means we won't accidentally traverse a symlink into some other # filesystem (which is potentially expensive to access). for i in range(len(parts)): prefix = os.sep.join(parts[:i + 1])
--- a/mercurial/posix.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/posix.py Sun Nov 27 20:44:52 2016 -0500 @@ -160,31 +160,88 @@ try: EXECFLAGS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - fh, fn = tempfile.mkstemp(dir=path, prefix='hg-checkexec-') + cachedir = os.path.join(path, '.hg', 'cache') + if os.path.isdir(cachedir): + checkisexec = os.path.join(cachedir, 'checkisexec') + checknoexec = os.path.join(cachedir, 'checknoexec') + + try: + m = os.stat(checkisexec).st_mode + except OSError as e: + if e.errno != errno.ENOENT: + raise + # checkisexec does not exist - fall through ... + else: + # checkisexec exists, check if it actually is exec + if m & EXECFLAGS != 0: + # ensure checkisexec exists, check it isn't exec + try: + m = os.stat(checknoexec).st_mode + except OSError as e: + if e.errno != errno.ENOENT: + raise + file(checknoexec, 'w').close() # might fail + m = os.stat(checknoexec).st_mode + if m & EXECFLAGS == 0: + # check-exec is exec and check-no-exec is not exec + return True + # checknoexec exists but is exec - delete it + os.unlink(checknoexec) + # checkisexec exists but is not exec - delete it + os.unlink(checkisexec) + + # check using one file, leave it as checkisexec + checkdir = cachedir + else: + # check directly in path and don't leave checkisexec behind + checkdir = path + checkisexec = None + fh, fn = tempfile.mkstemp(dir=checkdir, prefix='hg-checkexec-') try: os.close(fh) - m = os.stat(fn).st_mode & 0o777 - new_file_has_exec = m & EXECFLAGS - os.chmod(fn, m ^ EXECFLAGS) - exec_flags_cannot_flip = ((os.stat(fn).st_mode & 0o777) == m) + m = os.stat(fn).st_mode + if m & EXECFLAGS == 0: + os.chmod(fn, m & 0o777 | EXECFLAGS) + if os.stat(fn).st_mode & EXECFLAGS != 0: + if checkisexec is not None: + os.rename(fn, checkisexec) + fn = None + return True finally: - os.unlink(fn) + if fn is not None: + os.unlink(fn) except (IOError, OSError): # we don't care, the user probably won't be able to commit anyway return False - return not (new_file_has_exec or exec_flags_cannot_flip) def checklink(path): """check whether the given path is on a symlink-capable filesystem""" # mktemp is not racy because symlink creation will fail if the # file already exists while True: - name = tempfile.mktemp(dir=path, prefix='hg-checklink-') + cachedir = os.path.join(path, '.hg', 'cache') + checklink = os.path.join(cachedir, 'checklink') + # try fast path, read only + if os.path.islink(checklink): + return True + if os.path.isdir(cachedir): + checkdir = cachedir + else: + checkdir = path + cachedir = None + name = tempfile.mktemp(dir=checkdir, prefix='checklink-') try: - fd = tempfile.NamedTemporaryFile(dir=path, prefix='hg-checklink-') + fd = tempfile.NamedTemporaryFile(dir=checkdir, + prefix='hg-checklink-') try: os.symlink(os.path.basename(fd.name), name) - os.unlink(name) + if cachedir is None: + os.unlink(name) + else: + try: + os.rename(name, checklink) + except OSError: + os.unlink(name) return True except OSError as inst: # link creation might race, try again @@ -463,36 +520,6 @@ def gethgcmd(): return sys.argv[:1] -def termwidth(): - try: - import array - import termios - for dev in (sys.stderr, sys.stdout, sys.stdin): - try: - try: - fd = dev.fileno() - except AttributeError: - continue - if not os.isatty(fd): - continue - try: - arri = fcntl.ioctl(fd, termios.TIOCGWINSZ, '\0' * 8) - width = array.array('h', arri)[1] - if width > 0: - return width - except AttributeError: - pass - except ValueError: - pass - except IOError as e: - if e[0] == errno.EINVAL: - pass - else: - raise - except ImportError: - pass - return 80 - def makedir(path, notindexed): os.mkdir(path)
--- a/mercurial/profiling.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/profiling.py Sun Nov 27 20:44:52 2016 -0500 @@ -9,7 +9,6 @@ import contextlib import os -import sys import time from .i18n import _ @@ -80,11 +79,7 @@ @contextlib.contextmanager def statprofile(ui, fp): - try: - import statprof - except ImportError: - raise error.Abort(_( - 'statprof not available - install using "easy_install statprof"')) + from . import statprof freq = ui.configint('profiling', 'freq', default=1000) if freq > 0: @@ -94,12 +89,29 @@ else: ui.warn(_("invalid sampling frequency '%s' - ignoring\n") % freq) - statprof.start() + statprof.start(mechanism='thread') + try: yield finally: - statprof.stop() - statprof.display(fp) + data = statprof.stop() + + profformat = ui.config('profiling', 'statformat', 'hotpath') + + formats = { + 'byline': statprof.DisplayFormats.ByLine, + 'bymethod': statprof.DisplayFormats.ByMethod, + 'hotpath': statprof.DisplayFormats.Hotpath, + 'json': statprof.DisplayFormats.Json, + } + + if profformat in formats: + displayformat = formats[profformat] + else: + ui.warn(_('unknown profiler output format: %s\n') % profformat) + displayformat = statprof.DisplayFormats.Hotpath + + statprof.display(fp, data=data, format=displayformat) @contextlib.contextmanager def profile(ui): @@ -110,10 +122,10 @@ """ profiler = os.getenv('HGPROF') if profiler is None: - profiler = ui.config('profiling', 'type', default='ls') + profiler = ui.config('profiling', 'type', default='stat') if profiler not in ('ls', 'stat', 'flame'): ui.warn(_("unrecognized profiler '%s' - ignored\n") % profiler) - profiler = 'ls' + profiler = 'stat' output = ui.config('profiling', 'output') @@ -123,7 +135,7 @@ path = ui.expandpath(output) fp = open(path, 'wb') else: - fp = sys.stderr + fp = ui.ferr try: if profiler == 'ls':
--- a/mercurial/progress.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/progress.py Sun Nov 27 20:44:52 2016 -0500 @@ -7,7 +7,6 @@ from __future__ import absolute_import -import sys import threading import time @@ -19,7 +18,7 @@ def shouldprint(ui): return not (ui.quiet or ui.plain('progress')) and ( - ui._isatty(sys.stderr) or ui.configbool('progress', 'assume-tty')) + ui._isatty(ui.ferr) or ui.configbool('progress', 'assume-tty')) def fmtremaining(seconds): """format a number of remaining seconds in human readable way @@ -158,14 +157,14 @@ out = spacejoin(head, prog, tail) else: out = spacejoin(head, tail) - sys.stderr.write('\r' + encoding.trim(out, termwidth)) + self.ui.ferr.write('\r' + encoding.trim(out, termwidth)) self.lasttopic = topic - sys.stderr.flush() + self.ui.ferr.flush() def clear(self): if not self.printed or not self.lastprint or not shouldprint(self.ui): return - sys.stderr.write('\r%s\r' % (' ' * self.width())) + self.ui.ferr.write('\r%s\r' % (' ' * self.width())) if self.printed: # force immediate re-paint of progress bar self.lastprint = 0 @@ -176,8 +175,8 @@ if self.ui.configbool('progress', 'clear-complete', default=True): self.clear() else: - sys.stderr.write('\n') - sys.stderr.flush() + self.ui.ferr.write('\n') + self.ui.ferr.flush() def width(self): tw = self.ui.termwidth()
--- a/mercurial/pure/osutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/pure/osutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -14,7 +14,11 @@ import stat as statmod import sys -from . import policy +from . import ( + policy, + pycompat, +) + modulepolicy = policy.policy policynocffi = policy.policynocffi @@ -51,8 +55,8 @@ ''' result = [] prefix = path - if not prefix.endswith(os.sep): - prefix += os.sep + if not prefix.endswith(pycompat.ossep): + prefix += pycompat.ossep names = os.listdir(path) names.sort() for fn in names:
--- a/mercurial/pycompat.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/pycompat.py Sun Nov 27 20:44:52 2016 -0500 @@ -10,6 +10,7 @@ from __future__ import absolute_import +import os import sys ispy3 = (sys.version_info[0] >= 3) @@ -21,6 +22,7 @@ import Queue as _queue import SocketServer as socketserver import urlparse + urlunquote = urlparse.unquote import xmlrpclib else: import http.client as httplib @@ -29,13 +31,36 @@ import queue as _queue import socketserver import urllib.parse as urlparse + urlunquote = urlparse.unquote_to_bytes import xmlrpc.client as xmlrpclib if ispy3: import builtins import functools - import os fsencode = os.fsencode + fsdecode = os.fsdecode + # A bytes version of os.name. + osname = os.name.encode('ascii') + ospathsep = os.pathsep.encode('ascii') + ossep = os.sep.encode('ascii') + # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which + # returns bytes. + getcwd = os.getcwdb + + # TODO: .buffer might not exist if std streams were replaced; we'll need + # a silly wrapper to make a bytes stream backed by a unicode one. + stdin = sys.stdin.buffer + stdout = sys.stdout.buffer + stderr = sys.stderr.buffer + + # Since Python 3 converts argv to wchar_t type by Py_DecodeLocale() on Unix, + # we can use os.fsencode() to get back bytes argv. + # + # https://hg.python.org/cpython/file/v3.5.1/Programs/python.c#l55 + # + # TODO: On Windows, the native argv is wchar_t, so we'll need a different + # workaround to simulate the Python 2 (i.e. ANSI Win32 API) behavior. + sysargv = list(map(os.fsencode, sys.argv)) def sysstr(s): """Return a keyword str to be passed to Python functions such as @@ -76,6 +101,20 @@ raise TypeError( "expect str, not %s" % type(filename).__name__) + # In Python 2, fsdecode() has a very chance to receive bytes. So it's + # better not to touch Python 2 part as it's already working fine. + def fsdecode(filename): + return filename + + osname = os.name + ospathsep = os.pathsep + ossep = os.sep + stdin = sys.stdin + stdout = sys.stdout + stderr = sys.stderr + sysargv = sys.argv + getcwd = os.getcwd + stringio = io.StringIO empty = _queue.Empty queue = _queue.Queue
--- a/mercurial/registrar.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/registrar.py Sun Nov 27 20:44:52 2016 -0500 @@ -13,7 +13,7 @@ ) class _funcregistrarbase(object): - """Base of decorator to register a fuction for specific purpose + """Base of decorator to register a function for specific purpose This decorator stores decorated functions into own dict 'table'. @@ -177,7 +177,7 @@ Usage:: - templaetkeyword = registrar.templatekeyword() + templatekeyword = registrar.templatekeyword() @templatekeyword('mykeyword') def mykeywordfunc(repo, ctx, templ, cache, revcache, **args):
--- a/mercurial/repair.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/repair.py Sun Nov 27 20:44:52 2016 -0500 @@ -67,7 +67,7 @@ _, brokenset = revlog.getstrippoint(striprev) s.update([revlog.linkrev(r) for r in brokenset]) - collectone(repo.manifest) + collectone(repo.manifestlog._revlog) for fname in files: collectone(repo.file(fname)) @@ -153,7 +153,7 @@ tmpbundlefile = _bundle(repo, savebases, saveheads, node, 'temp', compress=False) - mfst = repo.manifest + mfst = repo.manifestlog._revlog curtr = repo.currenttransaction() if curtr is not None: @@ -174,7 +174,7 @@ if (unencoded.startswith('meta/') and unencoded.endswith('00manifest.i')): dir = unencoded[5:-12] - repo.manifest.dirlog(dir).strip(striprev, tr) + repo.manifestlog._revlog.dirlog(dir).strip(striprev, tr) for fn in files: repo.file(fn).strip(striprev, tr) tr.endgroup() @@ -244,6 +244,9 @@ vfs.unlink(tmpbundlefile) repo.destroyed() + # return the backup file path (or None if 'backup' was False) so + # extensions can use it + return backupfile def rebuildfncache(ui, repo): """Rebuilds the fncache file from repo history.
--- a/mercurial/revlog.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/revlog.py Sun Nov 27 20:44:52 2016 -0500 @@ -380,22 +380,29 @@ return r raise LookupError(node, self.indexfile, _('no node')) - def node(self, rev): - return self.index[rev][7] - def linkrev(self, rev): - return self.index[rev][4] - def parents(self, node): - i = self.index - d = i[self.rev(node)] - return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline - def parentrevs(self, rev): - return self.index[rev][5:7] + # Accessors for index entries. + + # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes + # are flags. def start(self, rev): return int(self.index[rev][0] >> 16) - def end(self, rev): - return self.start(rev) + self.length(rev) + + def flags(self, rev): + return self.index[rev][0] & 0xFFFF + def length(self, rev): return self.index[rev][1] + + def rawsize(self, rev): + """return the length of the uncompressed text for a given revision""" + l = self.index[rev][2] + if l >= 0: + return l + + t = self.revision(self.node(rev)) + return len(t) + size = rawsize + def chainbase(self, rev): base = self._chainbasecache.get(rev) if base is not None: @@ -409,6 +416,26 @@ self._chainbasecache[rev] = base return base + + def linkrev(self, rev): + return self.index[rev][4] + + def parentrevs(self, rev): + return self.index[rev][5:7] + + def node(self, rev): + return self.index[rev][7] + + # Derived from index values. + + def end(self, rev): + return self.start(rev) + self.length(rev) + + def parents(self, node): + i = self.index + d = i[self.rev(node)] + return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline + def chainlen(self, rev): return self._chaininfo(rev)[0] @@ -478,18 +505,6 @@ chain.reverse() return chain, stopped - def flags(self, rev): - return self.index[rev][0] & 0xFFFF - def rawsize(self, rev): - """return the length of the uncompressed text for a given revision""" - l = self.index[rev][2] - if l >= 0: - return l - - t = self.revision(self.node(rev)) - return len(t) - size = rawsize - def ancestors(self, revs, stoprev=0, inclusive=False): """Generate the ancestors of 'revs' in reverse topological order. Does not generate revs lower than stoprev. @@ -582,7 +597,7 @@ visit.append(p) missing = list(missing) missing.sort() - return has, [self.node(r) for r in missing] + return has, [self.node(miss) for miss in missing] def incrementalmissingrevs(self, common=None): """Return an object that can be used to incrementally compute the @@ -734,10 +749,10 @@ # include roots that aren't ancestors. # Filter out roots that aren't ancestors of heads - roots = [n for n in roots if n in ancestors] + roots = [root for root in roots if root in ancestors] # Recompute the lowest revision if roots: - lowestrev = min([self.rev(n) for n in roots]) + lowestrev = min([self.rev(root) for root in roots]) else: # No more roots? Return empty list return nonodes @@ -796,7 +811,7 @@ # But, obviously its parents aren't. for p in self.parents(n): heads.pop(p, None) - heads = [n for n, flag in heads.iteritems() if flag] + heads = [head for head, flag in heads.iteritems() if flag] roots = list(roots) assert orderedout assert roots @@ -948,9 +963,9 @@ def _partialmatch(self, id): try: - n = self.index.partialmatch(id) - if n and self.hasnode(n): - return n + partial = self.index.partialmatch(id) + if partial and self.hasnode(partial): + return partial return None except RevlogError: # parsers.c radix tree lookup gave multiple matches @@ -1094,8 +1109,17 @@ Callers will need to call ``self.start(rev)`` and ``self.length(rev)`` to determine where each revision's data begins and ends. """ - start = self.start(startrev) - end = self.end(endrev) + # Inlined self.start(startrev) & self.end(endrev) for perf reasons + # (functions are expensive). + index = self.index + istart = index[startrev] + start = int(istart[0] >> 16) + if startrev == endrev: + end = start + istart[1] + else: + iend = index[endrev] + end = int(iend[0] >> 16) + iend[1] + if self._inline: start += (startrev + 1) * self._io.size end += (endrev + 1) * self._io.size
--- a/mercurial/revset.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/revset.py Sun Nov 27 20:44:52 2016 -0500 @@ -437,10 +437,10 @@ def func(repo, subset, a, b, order): f = getsymbol(a) if f in symbols: - fn = symbols[f] - if getattr(fn, '_takeorder', False): - return fn(repo, subset, b, order) - return fn(repo, subset, b) + func = symbols[f] + if getattr(func, '_takeorder', False): + return func(repo, subset, b, order) + return func(repo, subset, b) keep = lambda fn: getattr(fn, '__doc__', None) is not None @@ -1443,7 +1443,7 @@ results = set(cl.findmissingrevs(common=exclude, heads=include)) # XXX we should turn this into a baseset instead of a set, smartset may do - # some optimisations from the fact this is a baseset. + # some optimizations from the fact this is a baseset. return subset & results @predicate('origin([set])', safe=True) @@ -1475,7 +1475,7 @@ o = set([_firstsrc(r) for r in dests]) o -= set([None]) # XXX we should turn this into a baseset instead of a set, smartset may do - # some optimisations from the fact this is a baseset. + # some optimizations from the fact this is a baseset. return subset & o @predicate('outgoing([path])', safe=True) @@ -1521,7 +1521,7 @@ ps.add(cl.parentrevs(r)[0]) ps -= set([node.nullrev]) # XXX we should turn this into a baseset instead of a set, smartset may do - # some optimisations from the fact this is a baseset. + # some optimizations from the fact this is a baseset. return subset & ps @predicate('p2([set])', safe=True) @@ -1544,7 +1544,7 @@ ps.add(cl.parentrevs(r)[1]) ps -= set([node.nullrev]) # XXX we should turn this into a baseset instead of a set, smartset may do - # some optimisations from the fact this is a baseset. + # some optimizations from the fact this is a baseset. return subset & ps def parentpost(repo, subset, x, order): @@ -3198,7 +3198,7 @@ def __len__(self): # Basic implementation to be changed in future patches. # until this gets improved, we use generator expression - # here, since list compr is free to call __len__ again + # here, since list comprehensions are free to call __len__ again # causing infinite recursion l = baseset(r for r in self) return len(l)
--- a/mercurial/scmposix.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/scmposix.py Sun Nov 27 20:44:52 2016 -0500 @@ -1,10 +1,15 @@ from __future__ import absolute_import +import array +import errno +import fcntl import os import sys from . import ( + encoding, osutil, + pycompat, ) def _rcfiles(path): @@ -26,7 +31,7 @@ root = 'etc/mercurial' # old mod_python does not set sys.argv if len(getattr(sys, 'argv', [])) > 0: - p = os.path.dirname(os.path.dirname(sys.argv[0])) + p = os.path.dirname(os.path.dirname(pycompat.sysargv[0])) if p != '/': path.extend(_rcfiles(os.path.join(p, root))) path.extend(_rcfiles('/' + root)) @@ -34,6 +39,34 @@ def userrcpath(): if sys.platform == 'plan9': - return [os.environ['home'] + '/lib/hgrc'] + return [encoding.environ['home'] + '/lib/hgrc'] else: return [os.path.expanduser('~/.hgrc')] + +def termsize(ui): + try: + import termios + TIOCGWINSZ = termios.TIOCGWINSZ # unavailable on IRIX (issue3449) + except (AttributeError, ImportError): + return 80, 24 + + for dev in (ui.ferr, ui.fout, ui.fin): + try: + try: + fd = dev.fileno() + except AttributeError: + continue + if not os.isatty(fd): + continue + arri = fcntl.ioctl(fd, TIOCGWINSZ, '\0' * 8) + height, width = array.array('h', arri)[:2] + if width > 0 and height > 0: + return width, height + except ValueError: + pass + except IOError as e: + if e[0] == errno.EINVAL: + pass + else: + raise + return 80, 24
--- a/mercurial/scmutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/scmutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -14,6 +14,7 @@ import os import re import shutil +import socket import stat import tempfile import threading @@ -27,6 +28,7 @@ osutil, pathutil, phases, + pycompat, revset, similar, util, @@ -39,6 +41,7 @@ systemrcpath = scmplatform.systemrcpath userrcpath = scmplatform.userrcpath +termsize = scmplatform.termsize class status(tuple): '''Named tuple with a list of files per status. The 'deleted', 'unknown' @@ -139,6 +142,108 @@ else: ui.status(_("no changes found\n")) +def callcatch(ui, func): + """call func() with global exception handling + + return func() if no exception happens. otherwise do some error handling + and return an exit code accordingly. does not handle all exceptions. + """ + try: + return func() + # Global exception handling, alphabetically + # Mercurial-specific first, followed by built-in and library exceptions + except error.LockHeld as inst: + if inst.errno == errno.ETIMEDOUT: + reason = _('timed out waiting for lock held by %s') % inst.locker + else: + reason = _('lock held by %s') % inst.locker + ui.warn(_("abort: %s: %s\n") % (inst.desc or inst.filename, reason)) + except error.LockUnavailable as inst: + ui.warn(_("abort: could not lock %s: %s\n") % + (inst.desc or inst.filename, inst.strerror)) + except error.OutOfBandError as inst: + if inst.args: + msg = _("abort: remote error:\n") + else: + msg = _("abort: remote error\n") + ui.warn(msg) + if inst.args: + ui.warn(''.join(inst.args)) + if inst.hint: + ui.warn('(%s)\n' % inst.hint) + except error.RepoError as inst: + ui.warn(_("abort: %s!\n") % inst) + if inst.hint: + ui.warn(_("(%s)\n") % inst.hint) + except error.ResponseError as inst: + ui.warn(_("abort: %s") % inst.args[0]) + if not isinstance(inst.args[1], basestring): + ui.warn(" %r\n" % (inst.args[1],)) + elif not inst.args[1]: + ui.warn(_(" empty string\n")) + else: + ui.warn("\n%r\n" % util.ellipsis(inst.args[1])) + except error.CensoredNodeError as inst: + ui.warn(_("abort: file censored %s!\n") % inst) + except error.RevlogError as inst: + ui.warn(_("abort: %s!\n") % inst) + except error.SignalInterrupt: + ui.warn(_("killed!\n")) + except error.InterventionRequired as inst: + ui.warn("%s\n" % inst) + if inst.hint: + ui.warn(_("(%s)\n") % inst.hint) + return 1 + except error.Abort as inst: + ui.warn(_("abort: %s\n") % inst) + if inst.hint: + ui.warn(_("(%s)\n") % inst.hint) + except ImportError as inst: + ui.warn(_("abort: %s!\n") % inst) + m = str(inst).split()[-1] + if m in "mpatch bdiff".split(): + ui.warn(_("(did you forget to compile extensions?)\n")) + elif m in "zlib".split(): + ui.warn(_("(is your Python install correct?)\n")) + except IOError as inst: + if util.safehasattr(inst, "code"): + ui.warn(_("abort: %s\n") % inst) + elif util.safehasattr(inst, "reason"): + try: # usually it is in the form (errno, strerror) + reason = inst.reason.args[1] + except (AttributeError, IndexError): + # it might be anything, for example a string + reason = inst.reason + if isinstance(reason, unicode): + # SSLError of Python 2.7.9 contains a unicode + reason = reason.encode(encoding.encoding, 'replace') + ui.warn(_("abort: error: %s\n") % reason) + elif (util.safehasattr(inst, "args") + and inst.args and inst.args[0] == errno.EPIPE): + pass + elif getattr(inst, "strerror", None): + if getattr(inst, "filename", None): + ui.warn(_("abort: %s: %s\n") % (inst.strerror, inst.filename)) + else: + ui.warn(_("abort: %s\n") % inst.strerror) + else: + raise + except OSError as inst: + if getattr(inst, "filename", None) is not None: + ui.warn(_("abort: %s: '%s'\n") % (inst.strerror, inst.filename)) + else: + ui.warn(_("abort: %s\n") % inst.strerror) + except MemoryError: + ui.warn(_("abort: out of memory\n")) + except SystemExit as inst: + # Commands shouldn't sys.exit directly, but give a return code. + # Just in case catch this and and pass exit code to caller. + return inst.code + except socket.error as inst: + ui.warn(_("abort: %s\n") % inst.args[-1]) + + return -1 + def checknewlabel(repo, lbl, kind): # Do not use the "kind" parameter in ui output. # It makes strings difficult to translate. @@ -754,7 +859,7 @@ if _rcpath is None: if 'HGRCPATH' in encoding.environ: _rcpath = [] - for p in os.environ['HGRCPATH'].split(os.pathsep): + for p in encoding.environ['HGRCPATH'].split(pycompat.ospathsep): if not p: continue p = util.expandpath(p)
--- a/mercurial/scmwindows.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/scmwindows.py Sun Nov 27 20:44:52 2016 -0500 @@ -5,6 +5,7 @@ from . import ( osutil, util, + win32, ) try: @@ -51,3 +52,6 @@ path.append(os.path.join(userprofile, 'mercurial.ini')) path.append(os.path.join(userprofile, '.hgrc')) return path + +def termsize(ui): + return win32.termsize()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/server.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,162 @@ +# server.py - utility and factory of server +# +# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import errno +import os +import sys +import tempfile + +from .i18n import _ + +from . import ( + chgserver, + commandserver, + error, + hgweb, + util, +) + +def runservice(opts, parentfn=None, initfn=None, runfn=None, logfile=None, + runargs=None, appendpid=False): + '''Run a command as a service.''' + + def writepid(pid): + if opts['pid_file']: + if appendpid: + mode = 'a' + else: + mode = 'w' + fp = open(opts['pid_file'], mode) + fp.write(str(pid) + '\n') + fp.close() + + if opts['daemon'] and not opts['daemon_postexec']: + # Signal child process startup with file removal + lockfd, lockpath = tempfile.mkstemp(prefix='hg-service-') + os.close(lockfd) + try: + if not runargs: + runargs = util.hgcmd() + sys.argv[1:] + runargs.append('--daemon-postexec=unlink:%s' % lockpath) + # Don't pass --cwd to the child process, because we've already + # changed directory. + for i in xrange(1, len(runargs)): + if runargs[i].startswith('--cwd='): + del runargs[i] + break + elif runargs[i].startswith('--cwd'): + del runargs[i:i + 2] + break + def condfn(): + return not os.path.exists(lockpath) + pid = util.rundetached(runargs, condfn) + if pid < 0: + raise error.Abort(_('child process failed to start')) + writepid(pid) + finally: + try: + os.unlink(lockpath) + except OSError as e: + if e.errno != errno.ENOENT: + raise + if parentfn: + return parentfn(pid) + else: + return + + if initfn: + initfn() + + if not opts['daemon']: + writepid(util.getpid()) + + if opts['daemon_postexec']: + try: + os.setsid() + except AttributeError: + pass + for inst in opts['daemon_postexec']: + if inst.startswith('unlink:'): + lockpath = inst[7:] + os.unlink(lockpath) + elif inst.startswith('chdir:'): + os.chdir(inst[6:]) + elif inst != 'none': + raise error.Abort(_('invalid value for --daemon-postexec: %s') + % inst) + util.hidewindow() + util.stdout.flush() + util.stderr.flush() + + nullfd = os.open(os.devnull, os.O_RDWR) + logfilefd = nullfd + if logfile: + logfilefd = os.open(logfile, os.O_RDWR | os.O_CREAT | os.O_APPEND) + os.dup2(nullfd, 0) + os.dup2(logfilefd, 1) + os.dup2(logfilefd, 2) + if nullfd not in (0, 1, 2): + os.close(nullfd) + if logfile and logfilefd not in (0, 1, 2): + os.close(logfilefd) + + if runfn: + return runfn() + +_cmdservicemap = { + 'chgunix': chgserver.chgunixservice, + 'pipe': commandserver.pipeservice, + 'unix': commandserver.unixforkingservice, +} + +def _createcmdservice(ui, repo, opts): + mode = opts['cmdserver'] + try: + return _cmdservicemap[mode](ui, repo, opts) + except KeyError: + raise error.Abort(_('unknown mode %s') % mode) + +def _createhgwebservice(ui, repo, opts): + # this way we can check if something was given in the command-line + if opts.get('port'): + opts['port'] = util.getport(opts.get('port')) + + alluis = set([ui]) + if repo: + baseui = repo.baseui + alluis.update([repo.baseui, repo.ui]) + else: + baseui = ui + webconf = opts.get('web_conf') or opts.get('webdir_conf') + if webconf: + # load server settings (e.g. web.port) to "copied" ui, which allows + # hgwebdir to reload webconf cleanly + servui = ui.copy() + servui.readconfig(webconf, sections=['web']) + alluis.add(servui) + else: + servui = ui + + optlist = ("name templates style address port prefix ipv6" + " accesslog errorlog certificate encoding") + for o in optlist.split(): + val = opts.get(o, '') + if val in (None, ''): # should check against default options instead + continue + for u in alluis: + u.setconfig("web", o, val, 'serve') + + app = hgweb.createapp(baseui, repo, webconf) + return hgweb.httpservice(servui, app, opts) + +def createservice(ui, repo, opts): + if opts["cmdserver"]: + return _createcmdservice(ui, repo, opts) + else: + return _createhgwebservice(ui, repo, opts)
--- a/mercurial/simplemerge.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/simplemerge.py Sun Nov 27 20:44:52 2016 -0500 @@ -19,7 +19,6 @@ from __future__ import absolute_import import os -import sys from .i18n import _ from . import ( @@ -275,7 +274,7 @@ def minimize(self, merge_regions): """Trim conflict regions of lines where A and B sides match. - Lines where both A and B have made the same changes at the begining + Lines where both A and B have made the same changes at the beginning or the end of each merge region are eliminated from the conflict region and are instead considered the same. """ @@ -441,7 +440,7 @@ opener = scmutil.opener(os.path.dirname(local)) out = opener(os.path.basename(local), "w", atomictemp=True) else: - out = sys.stdout + out = ui.fout m3 = Merge3Text(basetext, localtext, othertext) extrakwargs = {
--- a/mercurial/sshserver.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/sshserver.py Sun Nov 27 20:44:52 2016 -0500 @@ -68,13 +68,6 @@ def redirect(self): pass - def groupchunks(self, fh): - return iter(lambda: fh.read(4096), '') - - def compresschunks(self, chunks): - for chunk in chunks: - yield chunk - def sendresponse(self, v): self.fout.write("%d\n" % len(v)) self.fout.write(v) @@ -82,7 +75,13 @@ def sendstream(self, source): write = self.fout.write - for chunk in source.gen: + + if source.reader: + gen = iter(lambda: source.reader.read(4096), '') + else: + gen = source.gen + + for chunk in gen: write(chunk) self.fout.flush()
--- a/mercurial/sslutil.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/sslutil.py Sun Nov 27 20:44:52 2016 -0500 @@ -638,7 +638,7 @@ # According to RFC 2818 the most specific Common Name must # be used. if key == 'commonName': - # 'subject' entries are unicide. + # 'subject' entries are unicode. try: value = value.encode('ascii') except UnicodeEncodeError: @@ -764,7 +764,7 @@ return None def validatesocket(sock): - """Validate a socket meets security requiremnets. + """Validate a socket meets security requirements. The passed socket must have been created with ``wrapsocket()``. """
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/statprof.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,805 @@ +#!/usr/bin/env python +## statprof.py +## Copyright (C) 2012 Bryan O'Sullivan <bos@serpentine.com> +## Copyright (C) 2011 Alex Fraser <alex at phatcore dot com> +## Copyright (C) 2004,2005 Andy Wingo <wingo at pobox dot com> +## Copyright (C) 2001 Rob Browning <rlb at defaultvalue dot org> + +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## +## You should have received a copy of the GNU Lesser General Public +## License along with this program; if not, contact: +## +## Free Software Foundation Voice: +1-617-542-5942 +## 59 Temple Place - Suite 330 Fax: +1-617-542-2652 +## Boston, MA 02111-1307, USA gnu@gnu.org + +""" +statprof is intended to be a fairly simple statistical profiler for +python. It was ported directly from a statistical profiler for guile, +also named statprof, available from guile-lib [0]. + +[0] http://wingolog.org/software/guile-lib/statprof/ + +To start profiling, call statprof.start(): +>>> start() + +Then run whatever it is that you want to profile, for example: +>>> import test.pystone; test.pystone.pystones() + +Then stop the profiling and print out the results: +>>> stop() +>>> display() + % cumulative self + time seconds seconds name + 26.72 1.40 0.37 pystone.py:79:Proc0 + 13.79 0.56 0.19 pystone.py:133:Proc1 + 13.79 0.19 0.19 pystone.py:208:Proc8 + 10.34 0.16 0.14 pystone.py:229:Func2 + 6.90 0.10 0.10 pystone.py:45:__init__ + 4.31 0.16 0.06 pystone.py:53:copy + ... + +All of the numerical data is statistically approximate. In the +following column descriptions, and in all of statprof, "time" refers +to execution time (both user and system), not wall clock time. + +% time + The percent of the time spent inside the procedure itself (not + counting children). + +cumulative seconds + The total number of seconds spent in the procedure, including + children. + +self seconds + The total number of seconds spent in the procedure itself (not + counting children). + +name + The name of the procedure. + +By default statprof keeps the data collected from previous runs. If you +want to clear the collected data, call reset(): +>>> reset() + +reset() can also be used to change the sampling frequency from the +default of 1000 Hz. For example, to tell statprof to sample 50 times a +second: +>>> reset(50) + +This means that statprof will sample the call stack after every 1/50 of +a second of user + system time spent running on behalf of the python +process. When your process is idle (for example, blocking in a read(), +as is the case at the listener), the clock does not advance. For this +reason statprof is not currently not suitable for profiling io-bound +operations. + +The profiler uses the hash of the code object itself to identify the +procedures, so it won't confuse different procedures with the same name. +They will show up as two different rows in the output. + +Right now the profiler is quite simplistic. I cannot provide +call-graphs or other higher level information. What you see in the +table is pretty much all there is. Patches are welcome :-) + + +Threading +--------- + +Because signals only get delivered to the main thread in Python, +statprof only profiles the main thread. However because the time +reporting function uses per-process timers, the results can be +significantly off if other threads' work patterns are not similar to the +main thread's work patterns. +""" +# no-check-code +from __future__ import absolute_import, division, print_function + +import collections +import contextlib +import getopt +import inspect +import json +import os +import signal +import sys +import tempfile +import threading +import time + +defaultdict = collections.defaultdict +contextmanager = contextlib.contextmanager + +__all__ = ['start', 'stop', 'reset', 'display', 'profile'] + +skips = set(["util.py:check", "extensions.py:closure", + "color.py:colorcmd", "dispatch.py:checkargs", + "dispatch.py:<lambda>", "dispatch.py:_runcatch", + "dispatch.py:_dispatch", "dispatch.py:_runcommand", + "pager.py:pagecmd", "dispatch.py:run", + "dispatch.py:dispatch", "dispatch.py:runcommand", + "hg.py:<module>", "evolve.py:warnobserrors", + ]) + +########################################################################### +## Utils + +def clock(): + times = os.times() + return times[0] + times[1] + + +########################################################################### +## Collection data structures + +class ProfileState(object): + def __init__(self, frequency=None): + self.reset(frequency) + + def reset(self, frequency=None): + # total so far + self.accumulated_time = 0.0 + # start_time when timer is active + self.last_start_time = None + # a float + if frequency: + self.sample_interval = 1.0 / frequency + elif not hasattr(self, 'sample_interval'): + # default to 1000 Hz + self.sample_interval = 1.0 / 1000.0 + else: + # leave the frequency as it was + pass + self.remaining_prof_time = None + # for user start/stop nesting + self.profile_level = 0 + + self.samples = [] + + def accumulate_time(self, stop_time): + self.accumulated_time += stop_time - self.last_start_time + + def seconds_per_sample(self): + return self.accumulated_time / len(self.samples) + +state = ProfileState() + + +class CodeSite(object): + cache = {} + + __slots__ = ('path', 'lineno', 'function', 'source') + + def __init__(self, path, lineno, function): + self.path = path + self.lineno = lineno + self.function = function + self.source = None + + def __eq__(self, other): + try: + return (self.lineno == other.lineno and + self.path == other.path) + except: + return False + + def __hash__(self): + return hash((self.lineno, self.path)) + + @classmethod + def get(cls, path, lineno, function): + k = (path, lineno) + try: + return cls.cache[k] + except KeyError: + v = cls(path, lineno, function) + cls.cache[k] = v + return v + + def getsource(self, length): + if self.source is None: + lineno = self.lineno - 1 + fp = None + try: + fp = open(self.path) + for i, line in enumerate(fp): + if i == lineno: + self.source = line.strip() + break + except: + pass + finally: + if fp: + fp.close() + if self.source is None: + self.source = '' + + source = self.source + if len(source) > length: + source = source[:(length - 3)] + "..." + return source + + def filename(self): + return os.path.basename(self.path) + +class Sample(object): + __slots__ = ('stack', 'time') + + def __init__(self, stack, time): + self.stack = stack + self.time = time + + @classmethod + def from_frame(cls, frame, time): + stack = [] + + while frame: + stack.append(CodeSite.get(frame.f_code.co_filename, frame.f_lineno, + frame.f_code.co_name)) + frame = frame.f_back + + return Sample(stack, time) + +########################################################################### +## SIGPROF handler + +def profile_signal_handler(signum, frame): + if state.profile_level > 0: + now = clock() + state.accumulate_time(now) + + state.samples.append(Sample.from_frame(frame, state.accumulated_time)) + + signal.setitimer(signal.ITIMER_PROF, + state.sample_interval, 0.0) + state.last_start_time = now + +stopthread = threading.Event() +def samplerthread(tid): + while not stopthread.is_set(): + now = clock() + state.accumulate_time(now) + + frame = sys._current_frames()[tid] + state.samples.append(Sample.from_frame(frame, state.accumulated_time)) + + state.last_start_time = now + time.sleep(state.sample_interval) + + stopthread.clear() + +########################################################################### +## Profiling API + +def is_active(): + return state.profile_level > 0 + +lastmechanism = None +def start(mechanism='thread'): + '''Install the profiling signal handler, and start profiling.''' + state.profile_level += 1 + if state.profile_level == 1: + state.last_start_time = clock() + rpt = state.remaining_prof_time + state.remaining_prof_time = None + + global lastmechanism + lastmechanism = mechanism + + if mechanism == 'signal': + signal.signal(signal.SIGPROF, profile_signal_handler) + signal.setitimer(signal.ITIMER_PROF, + rpt or state.sample_interval, 0.0) + elif mechanism == 'thread': + frame = inspect.currentframe() + tid = [k for k, f in sys._current_frames().items() if f == frame][0] + state.thread = threading.Thread(target=samplerthread, + args=(tid,), name="samplerthread") + state.thread.start() + +def stop(): + '''Stop profiling, and uninstall the profiling signal handler.''' + state.profile_level -= 1 + if state.profile_level == 0: + if lastmechanism == 'signal': + rpt = signal.setitimer(signal.ITIMER_PROF, 0.0, 0.0) + signal.signal(signal.SIGPROF, signal.SIG_IGN) + state.remaining_prof_time = rpt[0] + elif lastmechanism == 'thread': + stopthread.set() + state.thread.join() + + state.accumulate_time(clock()) + state.last_start_time = None + statprofpath = os.environ.get('STATPROF_DEST') + if statprofpath: + save_data(statprofpath) + + return state + +def save_data(path): + with open(path, 'w+') as file: + file.write(str(state.accumulated_time) + '\n') + for sample in state.samples: + time = str(sample.time) + stack = sample.stack + sites = ['\1'.join([s.path, str(s.lineno), s.function]) + for s in stack] + file.write(time + '\0' + '\0'.join(sites) + '\n') + +def load_data(path): + lines = open(path, 'r').read().splitlines() + + state.accumulated_time = float(lines[0]) + state.samples = [] + for line in lines[1:]: + parts = line.split('\0') + time = float(parts[0]) + rawsites = parts[1:] + sites = [] + for rawsite in rawsites: + siteparts = rawsite.split('\1') + sites.append(CodeSite.get(siteparts[0], int(siteparts[1]), + siteparts[2])) + + state.samples.append(Sample(sites, time)) + + + +def reset(frequency=None): + '''Clear out the state of the profiler. Do not call while the + profiler is running. + + The optional frequency argument specifies the number of samples to + collect per second.''' + assert state.profile_level == 0, "Can't reset() while statprof is running" + CodeSite.cache.clear() + state.reset(frequency) + + +@contextmanager +def profile(): + start() + try: + yield + finally: + stop() + display() + + +########################################################################### +## Reporting API + +class SiteStats(object): + def __init__(self, site): + self.site = site + self.selfcount = 0 + self.totalcount = 0 + + def addself(self): + self.selfcount += 1 + + def addtotal(self): + self.totalcount += 1 + + def selfpercent(self): + return self.selfcount / len(state.samples) * 100 + + def totalpercent(self): + return self.totalcount / len(state.samples) * 100 + + def selfseconds(self): + return self.selfcount * state.seconds_per_sample() + + def totalseconds(self): + return self.totalcount * state.seconds_per_sample() + + @classmethod + def buildstats(cls, samples): + stats = {} + + for sample in samples: + for i, site in enumerate(sample.stack): + sitestat = stats.get(site) + if not sitestat: + sitestat = SiteStats(site) + stats[site] = sitestat + + sitestat.addtotal() + + if i == 0: + sitestat.addself() + + return [s for s in stats.itervalues()] + +class DisplayFormats: + ByLine = 0 + ByMethod = 1 + AboutMethod = 2 + Hotpath = 3 + FlameGraph = 4 + Json = 5 + +def display(fp=None, format=3, data=None, **kwargs): + '''Print statistics, either to stdout or the given file object.''' + data = data or state + + if fp is None: + import sys + fp = sys.stdout + if len(data.samples) == 0: + print('No samples recorded.', file=fp) + return + + if format == DisplayFormats.ByLine: + display_by_line(data, fp) + elif format == DisplayFormats.ByMethod: + display_by_method(data, fp) + elif format == DisplayFormats.AboutMethod: + display_about_method(data, fp, **kwargs) + elif format == DisplayFormats.Hotpath: + display_hotpath(data, fp, **kwargs) + elif format == DisplayFormats.FlameGraph: + write_to_flame(data, fp, **kwargs) + elif format == DisplayFormats.Json: + write_to_json(data, fp) + else: + raise Exception("Invalid display format") + + if format != DisplayFormats.Json: + print('---', file=fp) + print('Sample count: %d' % len(data.samples), file=fp) + print('Total time: %f seconds' % data.accumulated_time, file=fp) + +def display_by_line(data, fp): + '''Print the profiler data with each sample line represented + as one row in a table. Sorted by self-time per line.''' + stats = SiteStats.buildstats(data.samples) + stats.sort(reverse=True, key=lambda x: x.selfseconds()) + + print('%5.5s %10.10s %7.7s %-8.8s' % + ('% ', 'cumulative', 'self', ''), file=fp) + print('%5.5s %9.9s %8.8s %-8.8s' % + ("time", "seconds", "seconds", "name"), file=fp) + + for stat in stats: + site = stat.site + sitelabel = '%s:%d:%s' % (site.filename(), site.lineno, site.function) + print('%6.2f %9.2f %9.2f %s' % (stat.selfpercent(), + stat.totalseconds(), + stat.selfseconds(), + sitelabel), + file=fp) + +def display_by_method(data, fp): + '''Print the profiler data with each sample function represented + as one row in a table. Important lines within that function are + output as nested rows. Sorted by self-time per line.''' + print('%5.5s %10.10s %7.7s %-8.8s' % + ('% ', 'cumulative', 'self', ''), file=fp) + print('%5.5s %9.9s %8.8s %-8.8s' % + ("time", "seconds", "seconds", "name"), file=fp) + + stats = SiteStats.buildstats(data.samples) + + grouped = defaultdict(list) + for stat in stats: + grouped[stat.site.filename() + ":" + stat.site.function].append(stat) + + # compute sums for each function + functiondata = [] + for fname, sitestats in grouped.iteritems(): + total_cum_sec = 0 + total_self_sec = 0 + total_percent = 0 + for stat in sitestats: + total_cum_sec += stat.totalseconds() + total_self_sec += stat.selfseconds() + total_percent += stat.selfpercent() + + functiondata.append((fname, + total_cum_sec, + total_self_sec, + total_percent, + sitestats)) + + # sort by total self sec + functiondata.sort(reverse=True, key=lambda x: x[2]) + + for function in functiondata: + if function[3] < 0.05: + continue + print('%6.2f %9.2f %9.2f %s' % (function[3], # total percent + function[1], # total cum sec + function[2], # total self sec + function[0]), # file:function + file=fp) + function[4].sort(reverse=True, key=lambda i: i.selfseconds()) + for stat in function[4]: + # only show line numbers for significant locations (>1% time spent) + if stat.selfpercent() > 1: + source = stat.site.getsource(25) + stattuple = (stat.selfpercent(), stat.selfseconds(), + stat.site.lineno, source) + + print('%33.0f%% %6.2f line %s: %s' % (stattuple), file=fp) + +def display_about_method(data, fp, function=None, **kwargs): + if function is None: + raise Exception("Invalid function") + + filename = None + if ':' in function: + filename, function = function.split(':') + + relevant_samples = 0 + parents = {} + children = {} + + for sample in data.samples: + for i, site in enumerate(sample.stack): + if site.function == function and (not filename + or site.filename() == filename): + relevant_samples += 1 + if i != len(sample.stack) - 1: + parent = sample.stack[i + 1] + if parent in parents: + parents[parent] = parents[parent] + 1 + else: + parents[parent] = 1 + + if site in children: + children[site] = children[site] + 1 + else: + children[site] = 1 + + parents = [(parent, count) for parent, count in parents.iteritems()] + parents.sort(reverse=True, key=lambda x: x[1]) + for parent, count in parents: + print('%6.2f%% %s:%s line %s: %s' % + (count / relevant_samples * 100, parent.filename(), + parent.function, parent.lineno, parent.getsource(50)), file=fp) + + stats = SiteStats.buildstats(data.samples) + stats = [s for s in stats + if s.site.function == function and + (not filename or s.site.filename() == filename)] + + total_cum_sec = 0 + total_self_sec = 0 + total_self_percent = 0 + total_cum_percent = 0 + for stat in stats: + total_cum_sec += stat.totalseconds() + total_self_sec += stat.selfseconds() + total_self_percent += stat.selfpercent() + total_cum_percent += stat.totalpercent() + + print( + '\n %s:%s Total: %0.2fs (%0.2f%%) Self: %0.2fs (%0.2f%%)\n' % + ( + filename or '___', + function, + total_cum_sec, + total_cum_percent, + total_self_sec, + total_self_percent + ), file=fp) + + children = [(child, count) for child, count in children.iteritems()] + children.sort(reverse=True, key=lambda x: x[1]) + for child, count in children: + print(' %6.2f%% line %s: %s' % + (count / relevant_samples * 100, child.lineno, + child.getsource(50)), file=fp) + +def display_hotpath(data, fp, limit=0.05, **kwargs): + class HotNode(object): + def __init__(self, site): + self.site = site + self.count = 0 + self.children = {} + + def add(self, stack, time): + self.count += time + site = stack[0] + child = self.children.get(site) + if not child: + child = HotNode(site) + self.children[site] = child + + if len(stack) > 1: + i = 1 + # Skip boiler plate parts of the stack + while i < len(stack) and '%s:%s' % (stack[i].filename(), stack[i].function) in skips: + i += 1 + if i < len(stack): + child.add(stack[i:], time) + + root = HotNode(None) + lasttime = data.samples[0].time + for sample in data.samples: + root.add(sample.stack[::-1], sample.time - lasttime) + lasttime = sample.time + + def _write(node, depth, multiple_siblings): + site = node.site + visiblechildren = [c for c in node.children.itervalues() + if c.count >= (limit * root.count)] + if site: + indent = depth * 2 - 1 + filename = '' + function = '' + if len(node.children) > 0: + childsite = list(node.children.itervalues())[0].site + filename = (childsite.filename() + ':').ljust(15) + function = childsite.function + + # lots of string formatting + listpattern = ''.ljust(indent) +\ + ('\\' if multiple_siblings else '|') +\ + ' %4.1f%% %s %s' + liststring = listpattern % (node.count / root.count * 100, + filename, function) + codepattern = '%' + str(55 - len(liststring)) + 's %s: %s' + codestring = codepattern % ('line', site.lineno, site.getsource(30)) + + finalstring = liststring + codestring + childrensamples = sum([c.count for c in node.children.itervalues()]) + # Make frames that performed more than 10% of the operation red + if node.count - childrensamples > (0.1 * root.count): + finalstring = '\033[91m' + finalstring + '\033[0m' + # Make frames that didn't actually perform work dark grey + elif node.count - childrensamples == 0: + finalstring = '\033[90m' + finalstring + '\033[0m' + print(finalstring, file=fp) + + newdepth = depth + if len(visiblechildren) > 1 or multiple_siblings: + newdepth += 1 + + visiblechildren.sort(reverse=True, key=lambda x: x.count) + for child in visiblechildren: + _write(child, newdepth, len(visiblechildren) > 1) + + if root.count > 0: + _write(root, 0, False) + +def write_to_flame(data, fp, scriptpath=None, outputfile=None, **kwargs): + if scriptpath is None: + scriptpath = os.environ['HOME'] + '/flamegraph.pl' + if not os.path.exists(scriptpath): + print("error: missing %s" % scriptpath, file=fp) + print("get it here: https://github.com/brendangregg/FlameGraph", + file=fp) + return + + fd, path = tempfile.mkstemp() + + file = open(path, "w+") + + lines = {} + for sample in data.samples: + sites = [s.function for s in sample.stack] + sites.reverse() + line = ';'.join(sites) + if line in lines: + lines[line] = lines[line] + 1 + else: + lines[line] = 1 + + for line, count in lines.iteritems(): + file.write("%s %s\n" % (line, count)) + + file.close() + + if outputfile is None: + outputfile = '~/flamegraph.svg' + + os.system("perl ~/flamegraph.pl %s > %s" % (path, outputfile)) + print("Written to %s" % outputfile, file=fp) + +def write_to_json(data, fp): + samples = [] + + for sample in data.samples: + stack = [] + + for frame in sample.stack: + stack.append((frame.path, frame.lineno, frame.function)) + + samples.append((sample.time, stack)) + + print(json.dumps(samples), file=fp) + +def printusage(): + print(""" +The statprof command line allows you to inspect the last profile's results in +the following forms: + +usage: + hotpath [-l --limit percent] + Shows a graph of calls with the percent of time each takes. + Red calls take over 10%% of the total time themselves. + lines + Shows the actual sampled lines. + functions + Shows the samples grouped by function. + function [filename:]functionname + Shows the callers and callees of a particular function. + flame [-s --script-path] [-o --output-file path] + Writes out a flamegraph to output-file (defaults to ~/flamegraph.svg) + Requires that ~/flamegraph.pl exist. + (Specify alternate script path with --script-path.)""") + +def main(argv=None): + if argv is None: + argv = sys.argv + + if len(argv) == 1: + printusage() + return 0 + + displayargs = {} + + optstart = 2 + displayargs['function'] = None + if argv[1] == 'hotpath': + displayargs['format'] = DisplayFormats.Hotpath + elif argv[1] == 'lines': + displayargs['format'] = DisplayFormats.ByLine + elif argv[1] == 'functions': + displayargs['format'] = DisplayFormats.ByMethod + elif argv[1] == 'function': + displayargs['format'] = DisplayFormats.AboutMethod + displayargs['function'] = argv[2] + optstart = 3 + elif argv[1] == 'flame': + displayargs['format'] = DisplayFormats.FlameGraph + else: + printusage() + return 0 + + # process options + try: + opts, args = getopt.getopt(sys.argv[optstart:], "hl:f:o:p:", + ["help", "limit=", "file=", "output-file=", "script-path="]) + except getopt.error as msg: + print(msg) + printusage() + return 2 + + displayargs['limit'] = 0.05 + path = None + for o, value in opts: + if o in ("-l", "--limit"): + displayargs['limit'] = float(value) + elif o in ("-f", "--file"): + path = value + elif o in ("-o", "--output-file"): + displayargs['outputfile'] = value + elif o in ("-p", "--script-path"): + displayargs['scriptpath'] = value + elif o in ("-h", "help"): + printusage() + return 0 + else: + assert False, "unhandled option %s" % o + + load_data(path=path) + + display(**displayargs) + + return 0 + +if __name__ == "__main__": + sys.exit(main())
--- a/mercurial/store.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/store.py Sun Nov 27 20:44:52 2016 -0500 @@ -436,7 +436,7 @@ self.entries = set(decodedir(fp.read()).splitlines()) if '' in self.entries: fp.seek(0) - for n, line in enumerate(fp): + for n, line in enumerate(util.iterfile(fp)): if not line.rstrip('\n'): t = _('invalid entry in fncache, line %d') % (n + 1) raise error.Abort(t)
--- a/mercurial/streamclone.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/streamclone.py Sun Nov 27 20:44:52 2016 -0500 @@ -286,11 +286,11 @@ def consumev1(repo, fp, filecount, bytecount): """Apply the contents from version 1 of a streaming clone file handle. - This takes the output from "streamout" and applies it to the specified + This takes the output from "stream_out" and applies it to the specified repository. - Like "streamout," the status line added by the wire protocol is not handled - by this function. + Like "stream_out," the status line added by the wire protocol is not + handled by this function. """ with repo.lock(): repo.ui.status(_('%d files to transfer, %s of data\n') %
--- a/mercurial/templatekw.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/templatekw.py Sun Nov 27 20:44:52 2016 -0500 @@ -458,7 +458,8 @@ # just avoid crash, we might want to use the 'ff...' hash in future return args = args.copy() - args.update({'rev': repo.manifest.rev(mnode), 'node': hex(mnode)}) + args.update({'rev': repo.manifestlog._revlog.rev(mnode), + 'node': hex(mnode)}) return templ('manifest', **args) def shownames(namespace, **args):
--- a/mercurial/templater.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/templater.py Sun Nov 27 20:44:52 2016 -0500 @@ -792,7 +792,7 @@ @templatefunc('rstdoc(text, style)') def rstdoc(context, mapping, args): - """Format ReStructuredText.""" + """Format reStructuredText.""" if len(args) != 2: # i18n: "rstdoc" is a keyword raise error.ParseError(_("rstdoc expects two arguments"))
--- a/mercurial/ui.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/ui.py Sun Nov 27 20:44:52 2016 -0500 @@ -7,6 +7,7 @@ from __future__ import absolute_import +import contextlib import errno import getpass import inspect @@ -22,9 +23,11 @@ from . import ( config, + encoding, error, formatter, progress, + pycompat, scmutil, util, ) @@ -129,9 +132,9 @@ self.httppasswordmgrdb = src.httppasswordmgrdb else: - self.fout = sys.stdout - self.ferr = sys.stderr - self.fin = sys.stdin + self.fout = util.stdout + self.ferr = util.stderr + self.fin = util.stdin # shared read-only environment self.environ = os.environ @@ -175,7 +178,7 @@ def readconfig(self, filename, root=None, trust=False, sections=None, remap=None): try: - fp = open(filename) + fp = open(filename, u'rb') except IOError: if not sections: # ignore unless we were looking for something return @@ -225,7 +228,7 @@ if section in (None, 'paths'): # expand vars and ~ # translate paths relative to root (or home) into absolute paths - root = root or os.getcwd() + root = root or pycompat.getcwd() for c in self._tcfg, self._ucfg, self._ocfg: for n, p in c.items('paths'): # Ignore sub-options. @@ -569,9 +572,11 @@ - False if HGPLAIN is not set, or feature is in HGPLAINEXCEPT - True otherwise ''' - if 'HGPLAIN' not in os.environ and 'HGPLAINEXCEPT' not in os.environ: + if ('HGPLAIN' not in encoding.environ and + 'HGPLAINEXCEPT' not in encoding.environ): return False - exceptions = os.environ.get('HGPLAINEXCEPT', '').strip().split(',') + exceptions = encoding.environ.get('HGPLAINEXCEPT', + '').strip().split(',') if feature and exceptions: return feature not in exceptions return True @@ -584,13 +589,13 @@ If not found and ui.askusername is True, ask the user, else use ($LOGNAME or $USER or $LNAME or $USERNAME) + "@full.hostname". """ - user = os.environ.get("HGUSER") + user = encoding.environ.get("HGUSER") if user is None: user = self.config("ui", ["username", "user"]) if user is not None: user = os.path.expandvars(user) if user is None: - user = os.environ.get("EMAIL") + user = encoding.environ.get("EMAIL") if user is None and self.configbool("ui", "askusername"): user = self.prompt(_("enter a commit username:"), default=None) if user is None and not self.interactive(): @@ -733,7 +738,7 @@ is curses, the interface for histedit is text and the interface for selecting chunk is crecord (the best curses interface available). - Consider the following exemple: + Consider the following example: ui.interface = curses ui.interface.histedit = text @@ -814,12 +819,12 @@ def termwidth(self): '''how wide is the terminal in columns? ''' - if 'COLUMNS' in os.environ: + if 'COLUMNS' in encoding.environ: try: - return int(os.environ['COLUMNS']) + return int(encoding.environ['COLUMNS']) except ValueError: pass - return util.termwidth() + return scmutil.termsize(self)[0] def formatted(self): '''should formatted output be used? @@ -1075,10 +1080,10 @@ editor = 'E' else: editor = 'vi' - return (os.environ.get("HGEDITOR") or + return (encoding.environ.get("HGEDITOR") or self.config("ui", "editor") or - os.environ.get("VISUAL") or - os.environ.get("EDITOR", editor)) + encoding.environ.get("VISUAL") or + encoding.environ.get("EDITOR", editor)) @util.propertycache def _progbar(self): @@ -1190,6 +1195,25 @@ " update your code.)") % version self.develwarn(msg, stacklevel=2, config='deprec-warn') + @contextlib.contextmanager + def configoverride(self, overrides, source=""): + """Context manager for temporary config overrides + `overrides` must be a dict of the following structure: + {(section, name) : value}""" + backups = {} + try: + for (section, name), value in overrides.items(): + backups[(section, name)] = self.backupconfig(section, name) + self.setconfig(section, name, value, source) + yield + finally: + for __, backup in backups.items(): + self.restoreconfig(backup) + # just restoring ui.quiet config to the previous value is not enough + # as it does not update ui.quiet class member + if ('ui', 'quiet') in overrides: + self.fixconfig(section='ui') + class paths(dict): """Represents a collection of paths and their configs.
--- a/mercurial/unionrepo.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/unionrepo.py Sun Nov 27 20:44:52 2016 -0500 @@ -13,8 +13,6 @@ from __future__ import absolute_import -import os - from .i18n import _ from .node import nullid @@ -27,6 +25,7 @@ manifest, mdiff, pathutil, + pycompat, revlog, scmutil, util, @@ -152,18 +151,18 @@ def baserevdiff(self, rev1, rev2): return changelog.changelog.revdiff(self, rev1, rev2) -class unionmanifest(unionrevlog, manifest.manifest): +class unionmanifest(unionrevlog, manifest.manifestrevlog): def __init__(self, opener, opener2, linkmapper): - manifest.manifest.__init__(self, opener) - manifest2 = manifest.manifest(opener2) + manifest.manifestrevlog.__init__(self, opener) + manifest2 = manifest.manifestrevlog(opener2) unionrevlog.__init__(self, opener, self.indexfile, manifest2, linkmapper) def baserevision(self, nodeorrev): - return manifest.manifest.revision(self, nodeorrev) + return manifest.manifestrevlog.revision(self, nodeorrev) def baserevdiff(self, rev1, rev2): - return manifest.manifest.revdiff(self, rev1, rev2) + return manifest.manifestrevlog.revdiff(self, rev1, rev2) class unionfilelog(unionrevlog, filelog.filelog): def __init__(self, opener, path, opener2, linkmapper, repo): @@ -229,7 +228,7 @@ return unionpeer(self) def getcwd(self): - return os.getcwd() # always outside the repo + return pycompat.getcwd() # always outside the repo def instance(ui, path, create): if create: @@ -237,13 +236,13 @@ parentpath = ui.config("bundle", "mainreporoot", "") if not parentpath: # try to find the correct path to the working directory repo - parentpath = cmdutil.findrepo(os.getcwd()) + parentpath = cmdutil.findrepo(pycompat.getcwd()) if parentpath is None: parentpath = '' if parentpath: # Try to make the full path relative so we get a nice, short URL. # In particular, we don't want temp dir names in test outputs. - cwd = os.getcwd() + cwd = pycompat.getcwd() if parentpath == cwd: parentpath = '' else:
--- a/mercurial/util.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/util.py Sun Nov 27 20:44:52 2016 -0500 @@ -24,10 +24,12 @@ import hashlib import imp import os +import platform as pyplatform import re as remod import shutil import signal import socket +import stat import string import subprocess import sys @@ -46,28 +48,24 @@ pycompat, ) -for attr in ( - 'empty', - 'httplib', - 'httpserver', - 'pickle', - 'queue', - 'urlerr', - 'urlparse', - # we do import urlreq, but we do it outside the loop - #'urlreq', - 'stringio', - 'socketserver', - 'xmlrpclib', -): - a = pycompat.sysstr(attr) - globals()[a] = getattr(pycompat, a) - -# This line is to make pyflakes happy: +empty = pycompat.empty +httplib = pycompat.httplib +httpserver = pycompat.httpserver +pickle = pycompat.pickle +queue = pycompat.queue +socketserver = pycompat.socketserver +stderr = pycompat.stderr +stdin = pycompat.stdin +stdout = pycompat.stdout +stringio = pycompat.stringio +urlerr = pycompat.urlerr +urlparse = pycompat.urlparse urlreq = pycompat.urlreq +xmlrpclib = pycompat.xmlrpclib if os.name == 'nt': from . import windows as platform + stdout = platform.winstdout(pycompat.stdout) else: from . import posix as platform @@ -122,7 +120,6 @@ statfiles = getattr(osutil, 'statfiles', platform.statfiles) statisexec = platform.statisexec statislink = platform.statislink -termwidth = platform.termwidth testpid = platform.testpid umask = platform.umask unlink = platform.unlink @@ -938,6 +935,9 @@ else: datapath = os.path.dirname(__file__) +if not isinstance(datapath, bytes): + datapath = pycompat.fsencode(datapath) + i18n.setdatapath(datapath) _hgexecutable = None @@ -986,7 +986,7 @@ if environ is None: environ = {} try: - sys.stdout.flush() + stdout.flush() except Exception: pass def py2shell(val): @@ -1454,7 +1454,7 @@ def __eq__(self, old): try: # if ambiguity between stat of new and old file is - # avoided, comparision of size, ctime and mtime is enough + # avoided, comparison of size, ctime and mtime is enough # to exactly detect change of a file regardless of platform return (self.stat.st_size == old.stat.st_size and self.stat.st_ctime == old.stat.st_ctime and @@ -2206,6 +2206,78 @@ subsequent_indent=hangindent) return wrapper.fill(line).encode(encoding.encoding) +if (pyplatform.python_implementation() == 'CPython' and + sys.version_info < (3, 0)): + # There is an issue in CPython that some IO methods do not handle EINTR + # correctly. The following table shows what CPython version (and functions) + # are affected (buggy: has the EINTR bug, okay: otherwise): + # + # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0 + # -------------------------------------------------- + # fp.__iter__ | buggy | buggy | okay + # fp.read* | buggy | okay [1] | okay + # + # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo. + # + # Here we workaround the EINTR issue for fileobj.__iter__. Other methods + # like "read*" are ignored for now, as Python < 2.7.4 is a minority. + # + # Although we can workaround the EINTR issue for fp.__iter__, it is slower: + # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in + # CPython 2, because CPython 2 maintains an internal readahead buffer for + # fp.__iter__ but not other fp.read* methods. + # + # On modern systems like Linux, the "read" syscall cannot be interrupted + # when reading "fast" files like on-disk files. So the EINTR issue only + # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG) + # files approximately as "fast" files and use the fast (unsafe) code path, + # to minimize the performance impact. + if sys.version_info >= (2, 7, 4): + # fp.readline deals with EINTR correctly, use it as a workaround. + def _safeiterfile(fp): + return iter(fp.readline, '') + else: + # fp.read* are broken too, manually deal with EINTR in a stupid way. + # note: this may block longer than necessary because of bufsize. + def _safeiterfile(fp, bufsize=4096): + fd = fp.fileno() + line = '' + while True: + try: + buf = os.read(fd, bufsize) + except OSError as ex: + # os.read only raises EINTR before any data is read + if ex.errno == errno.EINTR: + continue + else: + raise + line += buf + if '\n' in buf: + splitted = line.splitlines(True) + line = '' + for l in splitted: + if l[-1] == '\n': + yield l + else: + line = l + if not buf: + break + if line: + yield line + + def iterfile(fp): + fastpath = True + if type(fp) is file: + fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode) + if fastpath: + return fp + else: + return _safeiterfile(fp) +else: + # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed. + def iterfile(fp): + return fp + def iterlines(iterator): for chunk in iterator: for line in chunk.splitlines(): @@ -2396,7 +2468,7 @@ _safechars = "!~*'()+" _safepchars = "/!~*'()+:\\" - _matchscheme = remod.compile(r'^[a-zA-Z0-9+.\-]+:').match + _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match def __init__(self, path, parsequery=True, parsefragment=True): # We slowly chomp away at path until we have only the path left @@ -2410,7 +2482,7 @@ path, self.fragment = path.split('#', 1) # special case for Windows drive letters and UNC paths - if hasdriveletter(path) or path.startswith(r'\\'): + if hasdriveletter(path) or path.startswith('\\\\'): self.path = path return @@ -2488,7 +2560,7 @@ 'path', 'fragment'): v = getattr(self, a) if v is not None: - setattr(self, a, pycompat.urlparse.unquote(v)) + setattr(self, a, pycompat.urlunquote(v)) def __repr__(self): attrs = [] @@ -2687,9 +2759,9 @@ finally: elapsed = time.time() - start _timenesting[0] -= indent - sys.stderr.write('%s%s: %s\n' % - (' ' * _timenesting[0], func.__name__, - timecount(elapsed))) + stderr.write('%s%s: %s\n' % + (' ' * _timenesting[0], func.__name__, + timecount(elapsed))) return wrapper _sizeunits = (('m', 2**20), ('k', 2**10), ('g', 2**30), @@ -2754,7 +2826,7 @@ else: yield line % (fnmax, fnln, func) -def debugstacktrace(msg='stacktrace', skip=0, f=sys.stderr, otherf=sys.stdout): +def debugstacktrace(msg='stacktrace', skip=0, f=stderr, otherf=stdout): '''Writes a message to f (stderr) with a nicely formatted stacktrace. Skips the 'skip' last entries. By default it will flush stdout first. It can be used everywhere and intentionally does not require an ui object. @@ -2812,32 +2884,6 @@ yield path[:pos] pos = path.rfind('/', 0, pos) -# compression utility - -class nocompress(object): - def compress(self, x): - return x - def flush(self): - return "" - -compressors = { - None: nocompress, - # lambda to prevent early import - 'BZ': lambda: bz2.BZ2Compressor(), - 'GZ': lambda: zlib.compressobj(), - } -# also support the old form by courtesies -compressors['UN'] = compressors[None] - -def _makedecompressor(decompcls): - def generator(f): - d = decompcls() - for chunk in filechunkiter(f): - yield d.decompress(chunk) - def func(fh): - return chunkbuffer(generator(fh)) - return func - class ctxmanager(object): '''A context manager for use in 'with' blocks to allow multiple contexts to be entered at once. This is both safer and more @@ -2898,20 +2944,302 @@ raise exc_val return received and suppressed -def _bz2(): - d = bz2.BZ2Decompressor() - # Bzip2 stream start with BZ, but we stripped it. - # we put it back for good measure. - d.decompress('BZ') - return d - -decompressors = {None: lambda fh: fh, - '_truncatedBZ': _makedecompressor(_bz2), - 'BZ': _makedecompressor(lambda: bz2.BZ2Decompressor()), - 'GZ': _makedecompressor(lambda: zlib.decompressobj()), - } -# also support the old form by courtesies -decompressors['UN'] = decompressors[None] +# compression code + +class compressormanager(object): + """Holds registrations of various compression engines. + + This class essentially abstracts the differences between compression + engines to allow new compression formats to be added easily, possibly from + extensions. + + Compressors are registered against the global instance by calling its + ``register()`` method. + """ + def __init__(self): + self._engines = {} + # Bundle spec human name to engine name. + self._bundlenames = {} + # Internal bundle identifier to engine name. + self._bundletypes = {} + + def __getitem__(self, key): + return self._engines[key] + + def __contains__(self, key): + return key in self._engines + + def __iter__(self): + return iter(self._engines.keys()) + + def register(self, engine): + """Register a compression engine with the manager. + + The argument must be a ``compressionengine`` instance. + """ + if not isinstance(engine, compressionengine): + raise ValueError(_('argument must be a compressionengine')) + + name = engine.name() + + if name in self._engines: + raise error.Abort(_('compression engine %s already registered') % + name) + + bundleinfo = engine.bundletype() + if bundleinfo: + bundlename, bundletype = bundleinfo + + if bundlename in self._bundlenames: + raise error.Abort(_('bundle name %s already registered') % + bundlename) + if bundletype in self._bundletypes: + raise error.Abort(_('bundle type %s already registered by %s') % + (bundletype, self._bundletypes[bundletype])) + + # No external facing name declared. + if bundlename: + self._bundlenames[bundlename] = name + + self._bundletypes[bundletype] = name + + self._engines[name] = engine + + @property + def supportedbundlenames(self): + return set(self._bundlenames.keys()) + + @property + def supportedbundletypes(self): + return set(self._bundletypes.keys()) + + def forbundlename(self, bundlename): + """Obtain a compression engine registered to a bundle name. + + Will raise KeyError if the bundle type isn't registered. + + Will abort if the engine is known but not available. + """ + engine = self._engines[self._bundlenames[bundlename]] + if not engine.available(): + raise error.Abort(_('compression engine %s could not be loaded') % + engine.name()) + return engine + + def forbundletype(self, bundletype): + """Obtain a compression engine registered to a bundle type. + + Will raise KeyError if the bundle type isn't registered. + + Will abort if the engine is known but not available. + """ + engine = self._engines[self._bundletypes[bundletype]] + if not engine.available(): + raise error.Abort(_('compression engine %s could not be loaded') % + engine.name()) + return engine + +compengines = compressormanager() + +class compressionengine(object): + """Base class for compression engines. + + Compression engines must implement the interface defined by this class. + """ + def name(self): + """Returns the name of the compression engine. + + This is the key the engine is registered under. + + This method must be implemented. + """ + raise NotImplementedError() + + def available(self): + """Whether the compression engine is available. + + The intent of this method is to allow optional compression engines + that may not be available in all installations (such as engines relying + on C extensions that may not be present). + """ + return True + + def bundletype(self): + """Describes bundle identifiers for this engine. + + If this compression engine isn't supported for bundles, returns None. + + If this engine can be used for bundles, returns a 2-tuple of strings of + the user-facing "bundle spec" compression name and an internal + identifier used to denote the compression format within bundles. To + exclude the name from external usage, set the first element to ``None``. + + If bundle compression is supported, the class must also implement + ``compressstream`` and `decompressorreader``. + """ + return None + + def compressstream(self, it, opts=None): + """Compress an iterator of chunks. + + The method receives an iterator (ideally a generator) of chunks of + bytes to be compressed. It returns an iterator (ideally a generator) + of bytes of chunks representing the compressed output. + + Optionally accepts an argument defining how to perform compression. + Each engine treats this argument differently. + """ + raise NotImplementedError() + + def decompressorreader(self, fh): + """Perform decompression on a file object. + + Argument is an object with a ``read(size)`` method that returns + compressed data. Return value is an object with a ``read(size)`` that + returns uncompressed data. + """ + raise NotImplementedError() + +class _zlibengine(compressionengine): + def name(self): + return 'zlib' + + def bundletype(self): + return 'gzip', 'GZ' + + def compressstream(self, it, opts=None): + opts = opts or {} + + z = zlib.compressobj(opts.get('level', -1)) + for chunk in it: + data = z.compress(chunk) + # Not all calls to compress emit data. It is cheaper to inspect + # here than to feed empty chunks through generator. + if data: + yield data + + yield z.flush() + + def decompressorreader(self, fh): + def gen(): + d = zlib.decompressobj() + for chunk in filechunkiter(fh): + while chunk: + # Limit output size to limit memory. + yield d.decompress(chunk, 2 ** 18) + chunk = d.unconsumed_tail + + return chunkbuffer(gen()) + +compengines.register(_zlibengine()) + +class _bz2engine(compressionengine): + def name(self): + return 'bz2' + + def bundletype(self): + return 'bzip2', 'BZ' + + def compressstream(self, it, opts=None): + opts = opts or {} + z = bz2.BZ2Compressor(opts.get('level', 9)) + for chunk in it: + data = z.compress(chunk) + if data: + yield data + + yield z.flush() + + def decompressorreader(self, fh): + def gen(): + d = bz2.BZ2Decompressor() + for chunk in filechunkiter(fh): + yield d.decompress(chunk) + + return chunkbuffer(gen()) + +compengines.register(_bz2engine()) + +class _truncatedbz2engine(compressionengine): + def name(self): + return 'bz2truncated' + + def bundletype(self): + return None, '_truncatedBZ' + + # We don't implement compressstream because it is hackily handled elsewhere. + + def decompressorreader(self, fh): + def gen(): + # The input stream doesn't have the 'BZ' header. So add it back. + d = bz2.BZ2Decompressor() + d.decompress('BZ') + for chunk in filechunkiter(fh): + yield d.decompress(chunk) + + return chunkbuffer(gen()) + +compengines.register(_truncatedbz2engine()) + +class _noopengine(compressionengine): + def name(self): + return 'none' + + def bundletype(self): + return 'none', 'UN' + + def compressstream(self, it, opts=None): + return it + + def decompressorreader(self, fh): + return fh + +compengines.register(_noopengine()) + +class _zstdengine(compressionengine): + def name(self): + return 'zstd' + + @propertycache + def _module(self): + # Not all installs have the zstd module available. So defer importing + # until first access. + try: + from . import zstd + # Force delayed import. + zstd.__version__ + return zstd + except ImportError: + return None + + def available(self): + return bool(self._module) + + def bundletype(self): + return 'zstd', 'ZS' + + def compressstream(self, it, opts=None): + opts = opts or {} + # zstd level 3 is almost always significantly faster than zlib + # while providing no worse compression. It strikes a good balance + # between speed and compression. + level = opts.get('level', 3) + + zstd = self._module + z = zstd.ZstdCompressor(level=level).compressobj() + for chunk in it: + data = z.compress(chunk) + if data: + yield data + + yield z.flush() + + def decompressorreader(self, fh): + zstd = self._module + dctx = zstd.ZstdDecompressor() + return chunkbuffer(dctx.read_from(fh)) + +compengines.register(_zstdengine()) # convenient shortcut dst = debugstacktrace
--- a/mercurial/verify.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/verify.py Sun Nov 27 20:44:52 2016 -0500 @@ -51,7 +51,7 @@ self.errors = 0 self.warnings = 0 self.havecl = len(repo.changelog) > 0 - self.havemf = len(repo.manifest) > 0 + self.havemf = len(repo.manifestlog._revlog) > 0 self.revlogv1 = repo.changelog.version != revlog.REVLOGV0 self.lrugetctx = util.lrucachefunc(repo.changectx) self.refersmf = False @@ -201,7 +201,8 @@ progress=None): repo = self.repo ui = self.ui - mf = self.repo.manifest.dirlog(dir) + mfl = self.repo.manifestlog + mf = mfl._revlog.dirlog(dir) if not dir: self.ui.status(_("checking manifests\n")) @@ -235,7 +236,8 @@ self.err(lr, _("%s not in changesets") % short(n), label) try: - for f, fn, fl in mf.readshallowdelta(n).iterentries(): + mfdelta = mfl.get(dir, n).readdelta(shallow=True) + for f, fn, fl in mfdelta.iterentries(): if not f: self.err(lr, _("entry without name in manifest")) elif f == "/dev/null": # ignore this in very old repos @@ -423,7 +425,7 @@ # cross-check if f in filenodes: - fns = [(lr, n) for n, lr in filenodes[f].iteritems()] + fns = [(v, k) for k, v in filenodes[f].iteritems()] for lr, node in sorted(fns): self.err(lr, _("manifest refers to unknown revision %s") % short(node), f)
--- a/mercurial/win32.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/win32.py Sun Nov 27 20:44:52 2016 -0500 @@ -347,23 +347,25 @@ pid = _kernel32.GetCurrentProcessId() _user32.EnumWindows(_WNDENUMPROC(callback), pid) -def termwidth(): +def termsize(): # cmd.exe does not handle CR like a unix console, the CR is # counted in the line length. On 80 columns consoles, if 80 # characters are written, the following CR won't apply on the # current line but on the new one. Keep room for it. - width = 79 + width = 80 - 1 + height = 25 # Query stderr to avoid problems with redirections screenbuf = _kernel32.GetStdHandle( _STD_ERROR_HANDLE) # don't close the handle returned if screenbuf is None or screenbuf == _INVALID_HANDLE_VALUE: - return width + return width, height csbi = _CONSOLE_SCREEN_BUFFER_INFO() if not _kernel32.GetConsoleScreenBufferInfo( screenbuf, ctypes.byref(csbi)): - return width - width = csbi.srWindow.Right - csbi.srWindow.Left - return width + return width, height + width = csbi.srWindow.Right - csbi.srWindow.Left # don't '+ 1' + height = csbi.srWindow.Bottom - csbi.srWindow.Top + 1 + return width, height def _1stchild(pid): '''return the 1st found child of the given pid
--- a/mercurial/windows.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/windows.py Sun Nov 27 20:44:52 2016 -0500 @@ -38,7 +38,6 @@ setsignalhandler = win32.setsignalhandler spawndetached = win32.spawndetached split = os.path.split -termwidth = win32.termwidth testpid = win32.testpid unlink = win32.unlink @@ -172,8 +171,6 @@ self.close() raise IOError(errno.EPIPE, 'Broken pipe') -sys.__stdout__ = sys.stdout = winstdout(sys.stdout) - def _is_win_9x(): '''return true if run on windows 95, 98 or me.''' try:
--- a/mercurial/wireproto.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/wireproto.py Sun Nov 27 20:44:52 2016 -0500 @@ -10,7 +10,6 @@ import hashlib import itertools import os -import sys import tempfile from .i18n import _ @@ -78,21 +77,6 @@ # """ # raise NotImplementedError() - def groupchunks(self, fh): - """Generator of chunks to send to the client. - - Some protocols may have compressed the contents. - """ - raise NotImplementedError() - - def compresschunks(self, chunks): - """Generator of possible compressed chunks to send to the client. - - This is like ``groupchunks()`` except it accepts a generator as - its argument. - """ - raise NotImplementedError() - class remotebatch(peer.batcher): '''batches the queued calls; uses as few roundtrips as possible''' def __init__(self, remote): @@ -529,10 +513,19 @@ """wireproto reply: binary stream The call was successful and the result is a stream. - Iterate on the `self.gen` attribute to retrieve chunks. + + Accepts either a generator or an object with a ``read(size)`` method. + + ``v1compressible`` indicates whether this data can be compressed to + "version 1" clients (technically: HTTP peers using + application/mercurial-0.1 media type). This flag should NOT be used on + new commands because new clients should support a more modern compression + mechanism. """ - def __init__(self, gen): + def __init__(self, gen=None, reader=None, v1compressible=False): self.gen = gen + self.reader = reader + self.v1compressible = v1compressible class pushres(object): """wireproto reply: success with simple integer return @@ -581,8 +574,8 @@ opts[k] = others[k] del others[k] if others: - sys.stderr.write("warning: %s ignored unexpected arguments %s\n" - % (cmd, ",".join(others))) + util.stderr.write("warning: %s ignored unexpected arguments %s\n" + % (cmd, ",".join(others))) return opts def bundle1allowed(repo, action): @@ -739,14 +732,14 @@ def changegroup(repo, proto, roots): nodes = decodelist(roots) cg = changegroupmod.changegroup(repo, nodes, 'serve') - return streamres(proto.groupchunks(cg)) + return streamres(reader=cg, v1compressible=True) @wireprotocommand('changegroupsubset', 'bases heads') def changegroupsubset(repo, proto, bases, heads): bases = decodelist(bases) heads = decodelist(heads) cg = changegroupmod.changegroupsubset(repo, bases, heads, 'serve') - return streamres(proto.groupchunks(cg)) + return streamres(reader=cg, v1compressible=True) @wireprotocommand('debugwireargs', 'one two *') def debugwireargs(repo, proto, one, two, others): @@ -781,7 +774,7 @@ return ooberror(bundle2required) chunks = exchange.getbundlechunks(repo, 'serve', **opts) - return streamres(proto.compresschunks(chunks)) + return streamres(gen=chunks, v1compressible=True) @wireprotocommand('heads') def heads(repo, proto): @@ -870,7 +863,7 @@ # LockError may be raised before the first result is yielded. Don't # emit output until we're sure we got the lock successfully. it = streamclone.generatev1wireproto(repo) - return streamres(getstream(it)) + return streamres(gen=getstream(it)) except error.LockError: return '2\n' @@ -900,7 +893,7 @@ if util.safehasattr(r, 'addpart'): # The return looks streamable, we are in the bundle2 case and # should return a stream. - return streamres(r.getchunks()) + return streamres(gen=r.getchunks()) return pushres(r) finally: @@ -913,11 +906,11 @@ try: raise except error.Abort: - # The old code we moved used sys.stderr directly. + # The old code we moved used util.stderr directly. # We did not change it to minimise code change. # This need to be moved to something proper. # Feel free to do it. - sys.stderr.write("abort: %s\n" % exc) + util.stderr.write("abort: %s\n" % exc) return pushres(0) except error.PushRaced: return pusherr(str(exc)) @@ -962,4 +955,4 @@ manargs, advargs)) except error.PushRaced as exc: bundler.newpart('error:pushraced', [('message', str(exc))]) - return streamres(bundler.getchunks()) + return streamres(gen=bundler.getchunks())
--- a/mercurial/worker.py Mon Nov 14 22:43:25 2016 +0100 +++ b/mercurial/worker.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,10 +11,13 @@ import os import signal import sys -import threading from .i18n import _ -from . import error +from . import ( + error, + scmutil, + util, +) def countcpus(): '''try to count the number of CPUs on the system''' @@ -85,25 +88,12 @@ workers = _numworkers(ui) oldhandler = signal.getsignal(signal.SIGINT) signal.signal(signal.SIGINT, signal.SIG_IGN) - pids, problem = [], [0] - for pargs in partition(args, workers): - pid = os.fork() - if pid == 0: - signal.signal(signal.SIGINT, oldhandler) - try: - os.close(rfd) - for i, item in func(*(staticargs + (pargs,))): - os.write(wfd, '%d %s\n' % (i, item)) - os._exit(0) - except KeyboardInterrupt: - os._exit(255) - # other exceptions are allowed to propagate, we rely - # on lock.py's pid checks to avoid release callbacks - pids.append(pid) - pids.reverse() - os.close(wfd) - fp = os.fdopen(rfd, 'rb', 0) + pids, problem = set(), [0] def killworkers(): + # unregister SIGCHLD handler as all children will be killed. This + # function shouldn't be interrupted by another SIGCHLD; otherwise pids + # could be updated while iterating, which would cause inconsistency. + signal.signal(signal.SIGCHLD, oldchldhandler) # if one worker bails, there's no good reason to wait for the rest for p in pids: try: @@ -111,24 +101,72 @@ except OSError as err: if err.errno != errno.ESRCH: raise - def waitforworkers(): - for _pid in pids: - st = _exitstatus(os.wait()[1]) + def waitforworkers(blocking=True): + for pid in pids.copy(): + p = st = 0 + while True: + try: + p, st = os.waitpid(pid, (0 if blocking else os.WNOHANG)) + break + except OSError as e: + if e.errno == errno.EINTR: + continue + elif e.errno == errno.ECHILD: + # child would already be reaped, but pids yet been + # updated (maybe interrupted just after waitpid) + pids.discard(pid) + break + else: + raise + if p: + pids.discard(p) + st = _exitstatus(st) if st and not problem[0]: problem[0] = st - killworkers() - t = threading.Thread(target=waitforworkers) - t.start() + def sigchldhandler(signum, frame): + waitforworkers(blocking=False) + if problem[0]: + killworkers() + oldchldhandler = signal.signal(signal.SIGCHLD, sigchldhandler) + for pargs in partition(args, workers): + pid = os.fork() + if pid == 0: + signal.signal(signal.SIGINT, oldhandler) + signal.signal(signal.SIGCHLD, oldchldhandler) + + def workerfunc(): + os.close(rfd) + for i, item in func(*(staticargs + (pargs,))): + os.write(wfd, '%d %s\n' % (i, item)) + + # make sure we use os._exit in all code paths. otherwise the worker + # may do some clean-ups which could cause surprises like deadlock. + # see sshpeer.cleanup for example. + try: + scmutil.callcatch(ui, workerfunc) + except KeyboardInterrupt: + os._exit(255) + except: # never return, therefore no re-raises + try: + ui.traceback() + finally: + os._exit(255) + else: + os._exit(0) + pids.add(pid) + os.close(wfd) + fp = os.fdopen(rfd, 'rb', 0) def cleanup(): signal.signal(signal.SIGINT, oldhandler) - t.join() + waitforworkers() + signal.signal(signal.SIGCHLD, oldchldhandler) status = problem[0] if status: if status < 0: os.kill(os.getpid(), -status) sys.exit(status) try: - for line in fp: + for line in util.iterfile(fp): l = line.split(' ', 1) yield int(l[0]), l[1][:-1] except: # re-raises
--- a/setup.py Mon Nov 14 22:43:25 2016 +0100 +++ b/setup.py Sun Nov 27 20:44:52 2016 -0500 @@ -67,6 +67,7 @@ from setuptools import setup else: from distutils.core import setup +from distutils.ccompiler import new_compiler from distutils.core import Command, Extension from distutils.dist import Distribution from distutils.command.build import build @@ -167,7 +168,8 @@ # to not use any hgrc files and do no localization. env = {'HGMODULEPOLICY': 'py', 'HGRCPATH': '', - 'LANGUAGE': 'C'} + 'LANGUAGE': 'C', + 'PATH': ''} # make pypi modules that use os.environ['PATH'] happy if 'LD_LIBRARY_PATH' in os.environ: env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH'] if 'SystemRoot' in os.environ: @@ -275,7 +277,30 @@ # too late for some cases return not self.pure and Distribution.has_ext_modules(self) +# This is ugly as a one-liner. So use a variable. +buildextnegops = dict(getattr(build_ext, 'negative_options', {})) +buildextnegops['no-zstd'] = 'zstd' + class hgbuildext(build_ext): + user_options = build_ext.user_options + [ + ('zstd', None, 'compile zstd bindings [default]'), + ('no-zstd', None, 'do not compile zstd bindings'), + ] + + boolean_options = build_ext.boolean_options + ['zstd'] + negative_opt = buildextnegops + + def initialize_options(self): + self.zstd = True + return build_ext.initialize_options(self) + + def build_extensions(self): + # Filter out zstd if disabled via argument. + if not self.zstd: + self.extensions = [e for e in self.extensions + if e.name != 'mercurial.zstd'] + + return build_ext.build_extensions(self) def build_extension(self, ext): try: @@ -318,14 +343,16 @@ if self.distribution.pure: self.distribution.ext_modules = [] elif self.distribution.cffi: - import setup_mpatch_cffi - import setup_bdiff_cffi - exts = [setup_mpatch_cffi.ffi.distutils_extension(), - setup_bdiff_cffi.ffi.distutils_extension()] + from mercurial.cffi import ( + bdiff, + mpatch, + ) + exts = [mpatch.ffi.distutils_extension(), + bdiff.ffi.distutils_extension()] # cffi modules go here if sys.platform == 'darwin': - import setup_osutil_cffi - exts.append(setup_osutil_cffi.ffi.distutils_extension()) + from mercurial.cffi import osutil + exts.append(osutil.ffi.distutils_extension()) self.distribution.ext_modules = exts else: h = os.path.join(get_python_inc(), 'Python.h') @@ -551,8 +578,14 @@ 'mercurial/compat.h', 'mercurial/util.h'] +osutil_cflags = [] osutil_ldflags = [] +# platform specific macros: HAVE_SETPROCTITLE +for plat, func in [(re.compile('freebsd'), 'setproctitle')]: + if plat.search(sys.platform) and hasfunction(new_compiler(), func): + osutil_cflags.append('-DHAVE_%s' % func.upper()) + if sys.platform == 'darwin': osutil_ldflags += ['-framework', 'ApplicationServices'] @@ -573,12 +606,17 @@ 'mercurial/pathencode.c'], depends=common_depends), Extension('mercurial.osutil', ['mercurial/osutil.c'], + extra_compile_args=osutil_cflags, extra_link_args=osutil_ldflags, depends=common_depends), Extension('hgext.fsmonitor.pywatchman.bser', ['hgext/fsmonitor/pywatchman/bser.c']), ] +sys.path.insert(0, 'contrib/python-zstandard') +import setup_zstd +extmodules.append(setup_zstd.get_c_extension('mercurial.zstd')) + try: from distutils import cygwinccompiler
--- a/setup_bdiff_cffi.py Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -from __future__ import absolute_import - -import cffi -import os - -ffi = cffi.FFI() -ffi.set_source("_bdiff_cffi", - open(os.path.join(os.path.join(os.path.dirname(__file__), 'mercurial'), - 'bdiff.c')).read(), include_dirs=['mercurial']) -ffi.cdef(""" -struct bdiff_line { - int hash, n, e; - ssize_t len; - const char *l; -}; - -struct bdiff_hunk; -struct bdiff_hunk { - int a1, a2, b1, b2; - struct bdiff_hunk *next; -}; - -int bdiff_splitlines(const char *a, ssize_t len, struct bdiff_line **lr); -int bdiff_diff(struct bdiff_line *a, int an, struct bdiff_line *b, int bn, - struct bdiff_hunk *base); -void bdiff_freehunks(struct bdiff_hunk *l); -void free(void*); -""") - -if __name__ == '__main__': - ffi.compile()
--- a/setup_mpatch_cffi.py Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,35 +0,0 @@ -from __future__ import absolute_import - -import cffi -import os - -ffi = cffi.FFI() -mpatch_c = os.path.join(os.path.join(os.path.dirname(__file__), 'mercurial', - 'mpatch.c')) -ffi.set_source("_mpatch_cffi", open(mpatch_c).read(), - include_dirs=["mercurial"]) -ffi.cdef(""" - -struct mpatch_frag { - int start, end, len; - const char *data; -}; - -struct mpatch_flist { - struct mpatch_frag *base, *head, *tail; -}; - -extern "Python" struct mpatch_flist* cffi_get_next_item(void*, ssize_t); - -int mpatch_decode(const char *bin, ssize_t len, struct mpatch_flist** res); -ssize_t mpatch_calcsize(size_t len, struct mpatch_flist *l); -void mpatch_lfree(struct mpatch_flist *a); -static int mpatch_apply(char *buf, const char *orig, size_t len, - struct mpatch_flist *l); -struct mpatch_flist *mpatch_fold(void *bins, - struct mpatch_flist* (*get_next_item)(void*, ssize_t), - ssize_t start, ssize_t end); -""") - -if __name__ == '__main__': - ffi.compile()
--- a/setup_osutil_cffi.py Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -from __future__ import absolute_import - -import cffi - -ffi = cffi.FFI() -ffi.set_source("_osutil_cffi", """ -#include <sys/attr.h> -#include <sys/vnode.h> -#include <unistd.h> -#include <fcntl.h> -#include <time.h> - -typedef struct val_attrs { - uint32_t length; - attribute_set_t returned; - attrreference_t name_info; - fsobj_type_t obj_type; - struct timespec mtime; - uint32_t accessmask; - off_t datalength; -} __attribute__((aligned(4), packed)) val_attrs_t; -""", include_dirs=['mercurial']) -ffi.cdef(''' - -typedef uint32_t attrgroup_t; - -typedef struct attrlist { - uint16_t bitmapcount; /* number of attr. bit sets in list */ - uint16_t reserved; /* (to maintain 4-byte alignment) */ - attrgroup_t commonattr; /* common attribute group */ - attrgroup_t volattr; /* volume attribute group */ - attrgroup_t dirattr; /* directory attribute group */ - attrgroup_t fileattr; /* file attribute group */ - attrgroup_t forkattr; /* fork attribute group */ - ...; -}; - -typedef struct attribute_set { - ...; -} attribute_set_t; - -typedef struct attrreference { - int attr_dataoffset; - int attr_length; - ...; -} attrreference_t; - -typedef int ... off_t; - -typedef struct val_attrs { - uint32_t length; - attribute_set_t returned; - attrreference_t name_info; - uint32_t obj_type; - struct timespec mtime; - uint32_t accessmask; - off_t datalength; - ...; -} val_attrs_t; - -/* the exact layout of the above struct will be figured out during build time */ - -typedef int ... time_t; - -typedef struct timespec { - time_t tv_sec; - ...; -}; - -int getattrlist(const char* path, struct attrlist * attrList, void * attrBuf, - size_t attrBufSize, unsigned int options); - -int getattrlistbulk(int dirfd, struct attrlist * attrList, void * attrBuf, - size_t attrBufSize, uint64_t options); - -#define ATTR_BIT_MAP_COUNT ... -#define ATTR_CMN_NAME ... -#define ATTR_CMN_OBJTYPE ... -#define ATTR_CMN_MODTIME ... -#define ATTR_CMN_ACCESSMASK ... -#define ATTR_CMN_ERROR ... -#define ATTR_CMN_RETURNED_ATTRS ... -#define ATTR_FILE_DATALENGTH ... - -#define VREG ... -#define VDIR ... -#define VLNK ... -#define VBLK ... -#define VCHR ... -#define VFIFO ... -#define VSOCK ... - -#define S_IFMT ... - -int open(const char *path, int oflag, int perm); -int close(int); - -#define O_RDONLY ... -''') - -if __name__ == '__main__': - ffi.compile()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/drawdag.py Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,311 @@ +# drawdag.py - convert ASCII revision DAG to actual changesets +# +# Copyright 2016 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. +""" +create changesets from an ASCII graph for testing purpose. + +For example, given the following input:: + + c d + |/ + b + | + a + +4 changesets and 4 local tags will be created. +`hg log -G -T "{rev} {desc} (tag: {tags})"` will output:: + + o 3 d (tag: d tip) + | + | o 2 c (tag: c) + |/ + o 1 b (tag: b) + | + o 0 a (tag: a) + +For root nodes (nodes without parents) in the graph, they can be revsets +pointing to existing nodes. The ASCII graph could also have disconnected +components with same names referring to the same changeset. + +Therefore, given the repo having the 4 changesets (and tags) above, with the +following ASCII graph as input:: + + foo bar bar foo + | / | | + ancestor(c,d) a baz + +The result (`hg log -G -T "{desc}"`) will look like:: + + o foo + |\ + +---o bar + | | | + | o | baz + | / + +---o d + | | + +---o c + | | + o | b + |/ + o a + +Note that if you take the above `hg log` output directly as input. It will work +as expected - the result would be an isomorphic graph:: + + o foo + |\ + | | o d + | |/ + | | o c + | |/ + | | o bar + | |/| + | o | b + | |/ + o / baz + / + o a + +This is because 'o' is specially handled in the input: instead of using 'o' as +the node name, the word to the right will be used. +""" +from __future__ import absolute_import, print_function + +import collections +import itertools + +from mercurial.i18n import _ +from mercurial import ( + cmdutil, + context, + error, + node, + scmutil, +) + +cmdtable = {} +command = cmdutil.command(cmdtable) + +_pipechars = '\\/+-|' +_nonpipechars = ''.join(chr(i) for i in xrange(33, 127) + if chr(i) not in _pipechars) + +def _isname(ch): + """char -> bool. return True if ch looks like part of a name, False + otherwise""" + return ch in _nonpipechars + +def _parseasciigraph(text): + """str -> {str : [str]}. convert the ASCII graph to edges""" + lines = text.splitlines() + edges = collections.defaultdict(list) # {node: []} + + def get(y, x): + """(int, int) -> char. give a coordinate, return the char. return a + space for anything out of range""" + if x < 0 or y < 0: + return ' ' + try: + return lines[y][x] + except IndexError: + return ' ' + + def getname(y, x): + """(int, int) -> str. like get(y, x) but concatenate left and right + parts. if name is an 'o', try to replace it to the right""" + result = '' + for i in itertools.count(0): + ch = get(y, x - i) + if not _isname(ch): + break + result = ch + result + for i in itertools.count(1): + ch = get(y, x + i) + if not _isname(ch): + break + result += ch + if result == 'o': + # special handling, find the name to the right + result = '' + for i in itertools.count(2): + ch = get(y, x + i) + if ch == ' ' or ch in _pipechars: + if result or x + i >= len(lines[y]): + break + else: + result += ch + return result or 'o' + return result + + def parents(y, x): + """(int, int) -> [str]. follow the ASCII edges at given position, + return a list of parents""" + visited = set([(y, x)]) + visit = [] + result = [] + + def follow(y, x, expected): + """conditionally append (y, x) to visit array, if it's a char + in excepted. 'o' in expected means an '_isname' test. + if '-' (or '+') is not in excepted, and get(y, x) is '-' (or '+'), + the next line (y + 1, x) will be checked instead.""" + ch = get(y, x) + if any(ch == c and c not in expected for c in '-+'): + y += 1 + return follow(y + 1, x, expected) + if ch in expected or ('o' in expected and _isname(ch)): + visit.append((y, x)) + + # -o- # starting point: + # /|\ # follow '-' (horizontally), and '/|\' (to the bottom) + follow(y + 1, x, '|') + follow(y + 1, x - 1, '/') + follow(y + 1, x + 1, '\\') + follow(y, x - 1, '-') + follow(y, x + 1, '-') + + while visit: + y, x = visit.pop() + if (y, x) in visited: + continue + visited.add((y, x)) + ch = get(y, x) + if _isname(ch): + result.append(getname(y, x)) + continue + elif ch == '|': + follow(y + 1, x, '/|o') + follow(y + 1, x - 1, '/') + follow(y + 1, x + 1, '\\') + elif ch == '+': + follow(y, x - 1, '-') + follow(y, x + 1, '-') + follow(y + 1, x - 1, '/') + follow(y + 1, x + 1, '\\') + follow(y + 1, x, '|') + elif ch == '\\': + follow(y + 1, x + 1, '\\|o') + elif ch == '/': + follow(y + 1, x - 1, '/|o') + elif ch == '-': + follow(y, x - 1, '-+o') + follow(y, x + 1, '-+o') + return result + + for y, line in enumerate(lines): + for x, ch in enumerate(line): + if ch == '#': # comment + break + if _isname(ch): + edges[getname(y, x)] += parents(y, x) + + return dict(edges) + +class simplefilectx(object): + def __init__(self, path, data): + self._data = data + self._path = path + + def data(self): + return self._data + + def path(self): + return self._path + + def renamed(self): + return None + + def flags(self): + return '' + +class simplecommitctx(context.committablectx): + def __init__(self, repo, name, parentctxs, added=None): + opts = { + 'changes': scmutil.status([], added or [], [], [], [], [], []), + 'date': '0 0', + 'extra': {'branch': 'default'}, + } + super(simplecommitctx, self).__init__(self, name, **opts) + self._repo = repo + self._name = name + self._parents = parentctxs + self._parents.sort(key=lambda c: c.node()) + while len(self._parents) < 2: + self._parents.append(repo[node.nullid]) + + def filectx(self, key): + return simplefilectx(key, self._name) + + def commit(self): + return self._repo.commitctx(self) + +def _walkgraph(edges): + """yield node, parents in topologically order""" + visible = set(edges.keys()) + remaining = {} # {str: [str]} + for k, vs in edges.iteritems(): + for v in vs: + if v not in remaining: + remaining[v] = [] + remaining[k] = vs[:] + while remaining: + leafs = [k for k, v in remaining.items() if not v] + if not leafs: + raise error.Abort(_('the graph has cycles')) + for leaf in sorted(leafs): + if leaf in visible: + yield leaf, edges[leaf] + del remaining[leaf] + for k, v in remaining.iteritems(): + if leaf in v: + v.remove(leaf) + +@command('debugdrawdag', []) +def debugdrawdag(ui, repo, **opts): + """read an ASCII graph from stdin and create changesets + + The ASCII graph is like what :hg:`log -G` outputs, with each `o` replaced + to the name of the node. The command will create dummy changesets and local + tags with those names to make the dummy changesets easier to be referred + to. + + If the name of a node is a single character 'o', It will be replaced by the + word to the right. This makes it easier to reuse + :hg:`log -G -T '{desc}'` outputs. + + For root (no parents) nodes, revset can be used to query existing repo. + Note that the revset cannot have confusing characters which can be seen as + the part of the graph edges, like `|/+-\`. + """ + text = ui.fin.read() + + # parse the graph and make sure len(parents) <= 2 for each node + edges = _parseasciigraph(text) + for k, v in edges.iteritems(): + if len(v) > 2: + raise error.Abort(_('%s: too many parents: %s') + % (k, ' '.join(v))) + + committed = {None: node.nullid} # {name: node} + + # for leaf nodes, try to find existing nodes in repo + for name, parents in edges.iteritems(): + if len(parents) == 0: + try: + committed[name] = scmutil.revsingle(repo, name) + except error.RepoLookupError: + pass + + # commit in topological order + for name, parents in _walkgraph(edges): + if name in committed: + continue + pctxs = [repo[committed[n]] for n in parents] + ctx = simplecommitctx(repo, name, pctxs, [name]) + n = ctx.commit() + committed[name] = n + repo.tag(name, n, message=None, user=None, date=None, local=True)
--- a/tests/dumbhttp.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/dumbhttp.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,7 +11,7 @@ import sys from mercurial import ( - cmdutil, + server, util, ) @@ -51,5 +51,5 @@ 'daemon': not options.foreground, 'daemon_postexec': options.daemon_postexec} service = simplehttpservice(options.host, options.port) - cmdutil.service(opts, initfn=service.init, runfn=service.run, - runargs=[sys.executable, __file__] + sys.argv[1:]) + server.runservice(opts, initfn=service.init, runfn=service.run, + runargs=[sys.executable, __file__] + sys.argv[1:])
--- a/tests/dummysmtpd.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/dummysmtpd.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,7 +11,7 @@ import sys from mercurial import ( - cmdutil, + server, sslutil, ui as uimod, ) @@ -75,8 +75,8 @@ dummysmtpsecureserver(addr, opts.certificate) log('listening at %s:%d\n' % addr) - cmdutil.service(vars(opts), initfn=init, runfn=run, - runargs=[sys.executable, __file__] + sys.argv[1:]) + server.runservice(vars(opts), initfn=init, runfn=run, + runargs=[sys.executable, __file__] + sys.argv[1:]) if __name__ == '__main__': main()
--- a/tests/failfilemerge.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/failfilemerge.py Sun Nov 27 20:44:52 2016 -0500 @@ -1,4 +1,4 @@ -# extension to emulate interupting filemerge._filemerge +# extension to emulate interrupting filemerge._filemerge from __future__ import absolute_import
--- a/tests/filterpyflakes.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/filterpyflakes.py Sun Nov 27 20:44:52 2016 -0500 @@ -7,55 +7,33 @@ import re import sys -def makekey(typeandline): - """ - for sorting lines by: msgtype, path/to/file, lineno, message - - typeandline is a sequence of a message type and the entire message line - the message line format is path/to/file:line: message - - >>> makekey((3, 'example.py:36: any message')) - (3, 'example.py', 36, ' any message') - >>> makekey((7, 'path/to/file.py:68: dummy message')) - (7, 'path/to/file.py', 68, ' dummy message') - >>> makekey((2, 'fn:88: m')) > makekey((2, 'fn:9: m')) - True - """ - - msgtype, line = typeandline - fname, line, message = line.split(":", 2) - # line as int for ordering 9 before 88 - return msgtype, fname, int(line), message - - lines = [] for line in sys.stdin: - # We whitelist tests (see more messages in pyflakes.messages) + # We blacklist tests that are too noisy for us pats = [ - (r"imported but unused", None), - (r"local variable '.*' is assigned to but never used", None), - (r"unable to detect undefined names", None), - (r"undefined name '.*'", - r"undefined name '(WindowsError|memoryview)'") - ] + r"undefined name '(WindowsError|memoryview)'", + r"redefinition of unused '[^']+' from line", + ] - for msgtype, (pat, excl) in enumerate(pats): - if re.search(pat, line) and (not excl or not re.search(excl, line)): + keep = True + for pat in pats: + if re.search(pat, line): + keep = False break # pattern matches - else: - continue # no pattern matched, next line - fn = line.split(':', 1)[0] - f = open(fn) - data = f.read() - f.close() - if 'no-' 'check-code' in data: - continue - lines.append((msgtype, line)) + if keep: + fn = line.split(':', 1)[0] + f = open(fn) + data = f.read() + f.close() + if 'no-' 'check-code' in data: + continue + lines.append(line) -for msgtype, line in sorted(lines, key=makekey): +for line in lines: sys.stdout.write(line) print() # self test of "undefined name" detection for other than 'memoryview' if False: + print(memoryview) print(undefinedname)
--- a/tests/hghave.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/hghave.py Sun Nov 27 20:44:52 2016 -0500 @@ -150,7 +150,7 @@ @check("darcs", "darcs client") def has_darcs(): - return matchoutput('darcs --version', br'2\.[2-9]', True) + return matchoutput('darcs --version', br'\b2\.([2-9]|\d{2})', True) @check("mtn", "monotone client (>= 1.0)") def has_mtn(): @@ -610,3 +610,12 @@ @check("unziplinks", "unzip(1) understands and extracts symlinks") def unzip_understands_symlinks(): return matchoutput('unzip --help', br'Info-ZIP') + +@check("zstd", "zstd Python module available") +def has_zstd(): + try: + import mercurial.zstd + mercurial.zstd.__version__ + return True + except ImportError: + return False
--- a/tests/run-tests.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/run-tests.py Sun Nov 27 20:44:52 2016 -0500 @@ -58,6 +58,11 @@ import socket import subprocess import sys +try: + import sysconfig +except ImportError: + # sysconfig doesn't exist in Python 2.6 + sysconfig = None import tempfile import threading import time @@ -818,6 +823,8 @@ offset = '' if i == 0 else '%s' % i env["HGPORT%s" % offset] = '%s' % (self._startport + i) env = os.environ.copy() + if sysconfig is not None: + env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') env['TESTTMP'] = self._testtmp env['HOME'] = self._testtmp # This number should match portneeded in _getport
--- a/tests/sitecustomize.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/sitecustomize.py Sun Nov 27 20:44:52 2016 -0500 @@ -4,11 +4,10 @@ if os.environ.get('COVERAGE_PROCESS_START'): try: import coverage - import random + import uuid - # uuid is better, but not available in Python 2.4. covpath = os.path.join(os.environ['COVERAGE_DIR'], - 'cov.%s' % random.randrange(0, 1000000000000)) + 'cov.%s' % uuid.uuid1()) cov = coverage.coverage(data_file=covpath, auto_data=True) cov._warn_no_data = False cov._warn_unimported_source = False
--- a/tests/test-ancestor.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-ancestor.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,7 +11,7 @@ from mercurial.node import nullrev from mercurial import ( ancestor, - commands, + debugcommands, hg, ui as uimod, util, @@ -226,7 +226,7 @@ # C version not available return - commands.debugbuilddag(u, repo, dag) + debugcommands.debugbuilddag(u, repo, dag) # Compare the results of the Python and C versions. This does not # include choosing a winner when more than one gca exists -- we make # sure both return exactly the same set of gcas.
--- a/tests/test-atomictempfile.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-atomictempfile.py Sun Nov 27 20:44:52 2016 -0500 @@ -68,7 +68,7 @@ repetition = 3 # repeat atomic write with checkambig=True, to examine - # whether st_mtime is advanced multiple times as expecetd + # whether st_mtime is advanced multiple times as expected for j in xrange(repetition): atomicwrite(True) newstat = os.stat(self._filename) @@ -77,7 +77,7 @@ continue # st_mtime should be advanced "repetition" times, because - # all atomicwrite() occured at same time (in sec) + # all atomicwrite() occurred at same time (in sec) self.assertTrue(newstat.st_mtime == ((oldstat.st_mtime + repetition) & 0x7fffffff)) # no more examination is needed, if assumption above is true
--- a/tests/test-backout.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-backout.t Sun Nov 27 20:44:52 2016 -0500 @@ -272,7 +272,7 @@ (1) update to REV1 (REV2 => REV1) (2) revert by REV1^1 -(3) commit backnig out revision (REV3) +(3) commit backing out revision (REV3) (4) update to REV2 (REV3 => REV2) (5) merge with REV3 (REV2 => REV2, REV3) @@ -287,7 +287,7 @@ > preupdate.visibility = sh $TESTTMP/checkvisibility.sh preupdate > EOF -("-m" is needed to avoid writing dirstte changes out at other than +("-m" is needed to avoid writing dirstate changes out at other than invocation of the hook to be examined) $ hg backout --merge -d '3 0' 1 --tool=true -m 'fixed comment'
--- a/tests/test-basic.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-basic.t Sun Nov 27 20:44:52 2016 -0500 @@ -6,7 +6,6 @@ defaults.shelve=--date "0 0" defaults.tag=-d "0 0" devel.all-warnings=true - extensions.chgserver= (?) largefiles.usercache=$TESTTMP/.cache/largefiles (glob) ui.slash=True ui.interactive=False
--- a/tests/test-bdiff.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-bdiff.py Sun Nov 27 20:44:52 2016 -0500 @@ -11,13 +11,12 @@ if d: c = mpatch.patches(a, [d]) if c != b: - print("***", repr(a), repr(b)) - print("bad:") - print(repr(c)[:200]) - print(repr(d)) + print("bad diff+patch result from\n %r to\n %r:" % (a, b)) + print("bdiff: %r" % d) + print("patched: %r" % c[:200]) def test(a, b): - print("***", repr(a), repr(b)) + print("test", repr(a), repr(b)) test1(a, b) test1(b, a) @@ -43,13 +42,21 @@ #issue1295 def showdiff(a, b): + print('showdiff(\n %r,\n %r):' % (a, b)) bin = bdiff.bdiff(a, b) pos = 0 + q = 0 while pos < len(bin): p1, p2, l = struct.unpack(">lll", bin[pos:pos + 12]) pos += 12 - print(p1, p2, repr(bin[pos:pos + l])) + if p1: + print('', repr(a[q:p1])) + print('', p1, p2, repr(a[p1:p2]), '->', repr(bin[pos:pos + l])) pos += l + q = p2 + if q < len(a): + print('', repr(a[q:])) + showdiff("x\n\nx\n\nx\n\nx\n\nz\n", "x\n\nx\n\ny\n\nx\n\nx\n\nz\n") showdiff("x\n\nx\n\nx\n\nx\n\nz\n", "x\n\nx\n\ny\n\nx\n\ny\n\nx\n\nz\n") # we should pick up abbbc. rather than bc.de as the longest match @@ -71,3 +78,17 @@ testfixws("", "", 0) print("done") + +print("Nice diff for a trivial change:") +showdiff( + ''.join('<%s\n-\n' % i for i in range(5)), + ''.join('>%s\n-\n' % i for i in range(5))) + +print("Diff 1 to 3 lines - preference for appending:") +showdiff('a\n', 'a\n' * 3) +print("Diff 1 to 5 lines - preference for appending:") +showdiff('a\n', 'a\n' * 5) +print("Diff 3 to 1 lines - preference for removing trailing lines:") +showdiff('a\n' * 3, 'a\n') +print("Diff 5 to 1 lines - preference for removing trailing lines:") +showdiff('a\n' * 5, 'a\n')
--- a/tests/test-bdiff.py.out Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-bdiff.py.out Sun Nov 27 20:44:52 2016 -0500 @@ -1,27 +1,82 @@ -*** 'a\nc\n\n\n\n' 'a\nb\n\n\n' -*** 'a\nb\nc\n' 'a\nc\n' -*** '' '' -*** 'a\nb\nc' 'a\nb\nc' -*** 'a\nb\nc\nd\n' 'a\nd\n' -*** 'a\nb\nc\nd\n' 'a\nc\ne\n' -*** 'a\nb\nc\n' 'a\nc\n' -*** 'a\n' 'c\na\nb\n' -*** 'a\n' '' -*** 'a\n' 'b\nc\n' -*** 'a\n' 'c\na\n' -*** '' 'adjfkjdjksdhfksj' -*** '' 'ab' -*** '' 'abc' -*** 'a' 'a' -*** 'ab' 'ab' -*** 'abc' 'abc' -*** 'a\n' 'a\n' -*** 'a\nb' 'a\nb' -6 6 'y\n\n' -6 6 'y\n\n' -9 9 'y\n\n' -0 0 'a\nb\nb\n' -12 12 'b\nc\n.\n' -16 18 '' +test 'a\nc\n\n\n\n' 'a\nb\n\n\n' +test 'a\nb\nc\n' 'a\nc\n' +test '' '' +test 'a\nb\nc' 'a\nb\nc' +test 'a\nb\nc\nd\n' 'a\nd\n' +test 'a\nb\nc\nd\n' 'a\nc\ne\n' +test 'a\nb\nc\n' 'a\nc\n' +test 'a\n' 'c\na\nb\n' +test 'a\n' '' +test 'a\n' 'b\nc\n' +test 'a\n' 'c\na\n' +test '' 'adjfkjdjksdhfksj' +test '' 'ab' +test '' 'abc' +test 'a' 'a' +test 'ab' 'ab' +test 'abc' 'abc' +test 'a\n' 'a\n' +test 'a\nb' 'a\nb' +showdiff( + 'x\n\nx\n\nx\n\nx\n\nz\n', + 'x\n\nx\n\ny\n\nx\n\nx\n\nz\n'): + 'x\n\nx\n\n' + 6 6 '' -> 'y\n\n' + 'x\n\nx\n\nz\n' +showdiff( + 'x\n\nx\n\nx\n\nx\n\nz\n', + 'x\n\nx\n\ny\n\nx\n\ny\n\nx\n\nz\n'): + 'x\n\nx\n\n' + 6 6 '' -> 'y\n\n' + 'x\n\n' + 9 9 '' -> 'y\n\n' + 'x\n\nz\n' +showdiff( + 'a\nb\nb\nb\nc\n.\nd\ne\n.\nf\n', + 'a\nb\nb\na\nb\nb\nb\nc\n.\nb\nc\n.\nd\ne\nf\n'): + 0 0 '' -> 'a\nb\nb\n' + 'a\nb\nb\nb\nc\n.\n' + 12 12 '' -> 'b\nc\n.\n' + 'd\ne\n' + 16 18 '.\n' -> '' + 'f\n' done done +Nice diff for a trivial change: +showdiff( + '<0\n-\n<1\n-\n<2\n-\n<3\n-\n<4\n-\n', + '>0\n-\n>1\n-\n>2\n-\n>3\n-\n>4\n-\n'): + 0 3 '<0\n' -> '>0\n' + '-\n' + 5 8 '<1\n' -> '>1\n' + '-\n' + 10 13 '<2\n' -> '>2\n' + '-\n' + 15 18 '<3\n' -> '>3\n' + '-\n' + 20 23 '<4\n' -> '>4\n' + '-\n' +Diff 1 to 3 lines - preference for appending: +showdiff( + 'a\n', + 'a\na\na\n'): + 'a\n' + 2 2 '' -> 'a\na\n' +Diff 1 to 5 lines - preference for appending: +showdiff( + 'a\n', + 'a\na\na\na\na\n'): + 'a\n' + 2 2 '' -> 'a\na\na\na\n' +Diff 3 to 1 lines - preference for removing trailing lines: +showdiff( + 'a\na\na\n', + 'a\n'): + 'a\n' + 2 6 'a\na\n' -> '' +Diff 5 to 1 lines - preference for removing trailing lines: +showdiff( + 'a\na\na\na\na\n', + 'a\n'): + 'a\n' + 2 10 'a\na\na\na\n' -> ''
--- a/tests/test-bookmarks-pushpull.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-bookmarks-pushpull.t Sun Nov 27 20:44:52 2016 -0500 @@ -550,7 +550,7 @@ $ cd .. -Test to show result of bookmarks comparision +Test to show result of bookmarks comparison $ mkdir bmcomparison $ cd bmcomparison
--- a/tests/test-bookmarks.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-bookmarks.t Sun Nov 27 20:44:52 2016 -0500 @@ -734,7 +734,7 @@ summary: 0 -no-op update doesn't deactive bookmarks +no-op update doesn't deactivate bookmarks $ hg bookmarks * four 3:9ba5f110a0b3
--- a/tests/test-bundle-type.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-bundle-type.t Sun Nov 27 20:44:52 2016 -0500 @@ -35,17 +35,21 @@ test bundle types - $ for t in "None" "bzip2" "gzip" "none-v2" "v2" "v1" "gzip-v1"; do - > echo % test bundle type $t - > hg init t$t + $ testbundle() { + > echo % test bundle type $1 + > hg init t$1 > cd t1 - > hg bundle -t $t ../b$t ../t$t - > f -q -B6 -D ../b$t; echo - > cd ../t$t - > hg debugbundle ../b$t - > hg debugbundle --spec ../b$t + > hg bundle -t $1 ../b$1 ../t$1 + > f -q -B6 -D ../b$1; echo + > cd ../t$1 + > hg debugbundle ../b$1 + > hg debugbundle --spec ../b$1 > echo > cd .. + > } + + $ for t in "None" "bzip2" "gzip" "none-v2" "v2" "v1" "gzip-v1"; do + > testbundle $t > done % test bundle type None searching for changes @@ -106,6 +110,38 @@ c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf gzip-v1 +#if zstd + + $ for t in "zstd" "zstd-v2"; do + > testbundle $t + > done + % test bundle type zstd + searching for changes + 1 changesets found + HG20\x00\x00 (esc) + Stream params: sortdict([('Compression', 'ZS')]) + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" + c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf + zstd-v2 + + % test bundle type zstd-v2 + searching for changes + 1 changesets found + HG20\x00\x00 (esc) + Stream params: sortdict([('Compression', 'ZS')]) + changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])" + c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf + zstd-v2 + +#else + +zstd is a valid engine but isn't available + + $ hg -R t1 bundle -a -t zstd irrelevant.hg + abort: compression engine zstd could not be loaded + [255] + +#endif test garbage file
--- a/tests/test-check-code.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-check-code.t Sun Nov 27 20:44:52 2016 -0500 @@ -7,7 +7,7 @@ New errors are not allowed. Warnings are strongly discouraged. (The writing "no-che?k-code" is for not skipping this file when checking.) - $ hg locate | sed 's-\\-/-g' | + $ hg locate -X contrib/python-zstandard | sed 's-\\-/-g' | > xargs "$check_code" --warnings --per-file=0 || false Skipping hgext/fsmonitor/pywatchman/__init__.py it has no-che?k-code (glob) Skipping hgext/fsmonitor/pywatchman/bser.c it has no-che?k-code (glob) @@ -17,3 +17,20 @@ Skipping i18n/polib.py it has no-che?k-code (glob) Skipping mercurial/httpclient/__init__.py it has no-che?k-code (glob) Skipping mercurial/httpclient/_readers.py it has no-che?k-code (glob) + Skipping mercurial/statprof.py it has no-che?k-code (glob) + +@commands in debugcommands.py should be in alphabetical order. + + >>> import re + >>> commands = [] + >>> with open('mercurial/debugcommands.py', 'rb') as fh: + ... for line in fh: + ... m = re.match("^@command\('([a-z]+)", line) + ... if m: + ... commands.append(m.group(1)) + >>> scommands = list(sorted(commands)) + >>> for i, command in enumerate(scommands): + ... if command != commands[i]: + ... print('commands in debugcommands.py not sorted; first differing ' + ... 'command is %s; expected %s' % (commands[i], command)) + ... break
--- a/tests/test-check-module-imports.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-check-module-imports.t Sun Nov 27 20:44:52 2016 -0500 @@ -159,6 +159,7 @@ $ hg locate 'set:**.py or grep(r"^#!.*?python")' \ > 'tests/**.t' \ > -X contrib/debugshell.py \ + > -X contrib/python-zstandard/ \ > -X contrib/win32/hgwebdir_wsgi.py \ > -X doc/gendoc.py \ > -X doc/hgmanpage.py \
--- a/tests/test-check-py3-commands.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-check-py3-commands.t Sun Nov 27 20:44:52 2016 -0500 @@ -9,6 +9,6 @@ > $PYTHON3 `which hg` $cmd 2>&1 2>&1 | tail -1 > done version - TypeError: str expected, not bytes + NameError: name 'basestring' is not defined debuginstall - TypeError: str expected, not bytes + NameError: name 'basestring' is not defined
--- a/tests/test-check-py3-compat.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-check-py3-compat.t Sun Nov 27 20:44:52 2016 -0500 @@ -4,6 +4,17 @@ $ cd "$TESTDIR"/.. $ hg files 'set:(**.py)' | sed 's|\\|/|g' | xargs python contrib/check-py3-compat.py + contrib/python-zstandard/setup.py not using absolute_import + contrib/python-zstandard/setup_zstd.py not using absolute_import + contrib/python-zstandard/tests/common.py not using absolute_import + contrib/python-zstandard/tests/test_cffi.py not using absolute_import + contrib/python-zstandard/tests/test_compressor.py not using absolute_import + contrib/python-zstandard/tests/test_data_structures.py not using absolute_import + contrib/python-zstandard/tests/test_decompressor.py not using absolute_import + contrib/python-zstandard/tests/test_estimate_sizes.py not using absolute_import + contrib/python-zstandard/tests/test_module_attributes.py not using absolute_import + contrib/python-zstandard/tests/test_roundtrip.py not using absolute_import + contrib/python-zstandard/tests/test_train_dictionary.py not using absolute_import hgext/fsmonitor/pywatchman/__init__.py not using absolute_import hgext/fsmonitor/pywatchman/__init__.py requires print_function hgext/fsmonitor/pywatchman/capabilities.py not using absolute_import @@ -21,7 +32,11 @@ hgext/fsmonitor/pywatchman/pybser.py: error importing: <ImportError> No module named 'pybser' (error at __init__.py:*) hgext/fsmonitor/watchmanclient.py: error importing: <ImportError> No module named 'pybser' (error at __init__.py:*) hgext/mq.py: error importing: <TypeError> __import__() argument 1 must be str, not bytes (error at extensions.py:*) - mercurial/scmwindows.py: error importing: <ImportError> No module named 'winreg' (error at scmwindows.py:*) + mercurial/cffi/bdiff.py: error importing: <ImportError> No module named 'mercurial.cffi' (error at check-py3-compat.py:*) + mercurial/cffi/mpatch.py: error importing: <ImportError> No module named 'mercurial.cffi' (error at check-py3-compat.py:*) + mercurial/cffi/osutil.py: error importing: <ImportError> No module named 'mercurial.cffi' (error at check-py3-compat.py:*) + mercurial/scmwindows.py: error importing: <ImportError> No module named 'msvcrt' (error at win32.py:*) + mercurial/statprof.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at statprof.py:*) mercurial/win32.py: error importing: <ImportError> No module named 'msvcrt' (error at win32.py:*) mercurial/windows.py: error importing: <ImportError> No module named 'msvcrt' (error at windows.py:*)
--- a/tests/test-check-pyflakes.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-check-pyflakes.t Sun Nov 27 20:44:52 2016 -0500 @@ -10,6 +10,6 @@ > -X mercurial/pycompat.py \ > 2>/dev/null \ > | xargs pyflakes 2>/dev/null | "$TESTDIR/filterpyflakes.py" - tests/filterpyflakes.py:61: undefined name 'undefinedname' + contrib/python-zstandard/tests/test_data_structures.py:107: local variable 'size' is assigned to but never used + tests/filterpyflakes.py:39: undefined name 'undefinedname' -
--- a/tests/test-clone.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-clone.t Sun Nov 27 20:44:52 2016 -0500 @@ -31,6 +31,9 @@ default 10:a7949464abda $ ls .hg/cache branch2-served + checkisexec + checklink + checknoexec rbc-names-v1 rbc-revs-v1 @@ -45,6 +48,8 @@ $ ls .hg/cache branch2-served + checkisexec + checklink $ cat a a
--- a/tests/test-commandserver.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-commandserver.t Sun Nov 27 20:44:52 2016 -0500 @@ -135,6 +135,19 @@ summary: 1 +check that "histedit --commands=-" can read rules from the input channel: + + >>> import cStringIO + >>> from hgclient import readchannel, runcommand, check + >>> @check + ... def serverinput(server): + ... readchannel(server) + ... rules = 'pick eff892de26ec\n' + ... runcommand(server, ['histedit', '0', '--commands=-', + ... '--config', 'extensions.histedit='], + ... input=cStringIO.StringIO(rules)) + *** runcommand histedit 0 --commands=- --config extensions.histedit= + check that --cwd doesn't persist between requests: $ mkdir foo @@ -223,8 +236,6 @@ ... 'id'], ... input=stringio('some input')) *** runcommand --config hooks.pre-identify=python:hook.hook id - hook talking - now try to read something: 'some input' eff892de26ec tip $ rm hook.py* @@ -804,7 +815,7 @@ $ echo foo > foo $ hg add foo -(failuer before finalization) +(failure before finalization) >>> from hgclient import readchannel, runcommand, check >>> @check @@ -823,7 +834,7 @@ *** runcommand log *** runcommand verify -q -(failuer after finalization) +(failure after finalization) >>> from hgclient import readchannel, runcommand, check >>> @check
--- a/tests/test-commit-amend.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-commit-amend.t Sun Nov 27 20:44:52 2016 -0500 @@ -638,7 +638,7 @@ (no more unresolved files) $ hg ci -m 'merge bar' $ hg log --config diff.git=1 -pr . - changeset: 23:69c24fe01e35 + changeset: 23:163cfd7219f7 tag: tip parent: 22:30d96aeaf27b parent: 21:1aa437659d19 @@ -657,7 +657,7 @@ dd +======= +cc - +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc + +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc diff --git a/z b/zz rename from z rename to zz @@ -671,7 +671,7 @@ $ HGEDITOR="sh .hg/checkeditform.sh" hg ci --amend -m 'merge bar (amend message)' --edit HGEDITFORM=commit.amend.merge $ hg log --config diff.git=1 -pr . - changeset: 24:cfa2fbef3169 + changeset: 24:bca52d4ed186 tag: tip parent: 22:30d96aeaf27b parent: 21:1aa437659d19 @@ -690,7 +690,7 @@ dd +======= +cc - +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc + +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc diff --git a/z b/zz rename from z rename to zz @@ -704,7 +704,7 @@ $ hg mv zz z $ hg ci --amend -m 'merge bar (undo rename)' $ hg log --config diff.git=1 -pr . - changeset: 26:c34de68b014c + changeset: 26:12594a98ca3f tag: tip parent: 22:30d96aeaf27b parent: 21:1aa437659d19 @@ -723,7 +723,7 @@ dd +======= +cc - +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc + +>>>>>>> merge rev: 1aa437659d19 bar - test: aazzcc $ hg debugrename z z not renamed @@ -740,9 +740,9 @@ $ echo aa >> aaa $ hg ci -m 'merge bar again' $ hg log --config diff.git=1 -pr . - changeset: 28:37d40dcef03b + changeset: 28:dffde028b388 tag: tip - parent: 26:c34de68b014c + parent: 26:12594a98ca3f parent: 27:4c94d5bc65f5 user: test date: Thu Jan 01 00:00:00 1970 +0000 @@ -775,9 +775,9 @@ $ hg mv aaa aa $ hg ci --amend -m 'merge bar again (undo rename)' $ hg log --config diff.git=1 -pr . - changeset: 30:537c6d1b3633 + changeset: 30:18e3ba160489 tag: tip - parent: 26:c34de68b014c + parent: 26:12594a98ca3f parent: 27:4c94d5bc65f5 user: test date: Thu Jan 01 00:00:00 1970 +0000 @@ -817,9 +817,9 @@ use (c)hanged version, (d)elete, or leave (u)nresolved? c $ hg ci -m 'merge bar (with conflicts)' $ hg log --config diff.git=1 -pr . - changeset: 33:7afcba911942 + changeset: 33:b4c3035e2544 tag: tip - parent: 32:6075d69d215d + parent: 32:4b216ca5ba97 parent: 31:67db8847a540 user: test date: Thu Jan 01 00:00:00 1970 +0000 @@ -829,9 +829,9 @@ $ hg rm aa $ hg ci --amend -m 'merge bar (with conflicts, amended)' $ hg log --config diff.git=1 -pr . - changeset: 35:376965e47ddd + changeset: 35:1205ed810051 tag: tip - parent: 32:6075d69d215d + parent: 32:4b216ca5ba97 parent: 31:67db8847a540 user: test date: Thu Jan 01 00:00:00 1970 +0000 @@ -927,7 +927,7 @@ HG: M: HG: A: foo HG: R: - HG: diff -r 376965e47ddd foo + HG: diff -r 1205ed810051 foo HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/foo Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ @@ -941,12 +941,12 @@ HG: M: HG: A: foo y HG: R: - HG: diff -r 376965e47ddd foo + HG: diff -r 1205ed810051 foo HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/foo Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ HG: +foo - HG: diff -r 376965e47ddd y + HG: diff -r 1205ed810051 y HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/y Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ @@ -959,18 +959,18 @@ HG: M: HG: A: foo y HG: R: a - HG: diff -r 376965e47ddd a + HG: diff -r 1205ed810051 a HG: --- a/a Thu Jan 01 00:00:00 1970 +0000 HG: +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: @@ -1,2 +0,0 @@ HG: -a HG: -a - HG: diff -r 376965e47ddd foo + HG: diff -r 1205ed810051 foo HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/foo Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ HG: +foo - HG: diff -r 376965e47ddd y + HG: diff -r 1205ed810051 y HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/y Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ @@ -983,23 +983,23 @@ HG: M: HG: A: foo y HG: R: a x - HG: diff -r 376965e47ddd a + HG: diff -r 1205ed810051 a HG: --- a/a Thu Jan 01 00:00:00 1970 +0000 HG: +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: @@ -1,2 +0,0 @@ HG: -a HG: -a - HG: diff -r 376965e47ddd foo + HG: diff -r 1205ed810051 foo HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/foo Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ HG: +foo - HG: diff -r 376965e47ddd x + HG: diff -r 1205ed810051 x HG: --- a/x Thu Jan 01 00:00:00 1970 +0000 HG: +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: @@ -1,1 +0,0 @@ HG: -x - HG: diff -r 376965e47ddd y + HG: diff -r 1205ed810051 y HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/y Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ @@ -1014,23 +1014,23 @@ HG: M: HG: A: foo y HG: R: a x - HG: diff -r 376965e47ddd a + HG: diff -r 1205ed810051 a HG: --- a/a Thu Jan 01 00:00:00 1970 +0000 HG: +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: @@ -1,2 +0,0 @@ HG: -a HG: -a - HG: diff -r 376965e47ddd foo + HG: diff -r 1205ed810051 foo HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/foo Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@ HG: +foo - HG: diff -r 376965e47ddd x + HG: diff -r 1205ed810051 x HG: --- a/x Thu Jan 01 00:00:00 1970 +0000 HG: +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: @@ -1,1 +0,0 @@ HG: -x - HG: diff -r 376965e47ddd y + HG: diff -r 1205ed810051 y HG: --- /dev/null Thu Jan 01 00:00:00 1970 +0000 HG: +++ b/y Thu Jan 01 00:00:00 1970 +0000 HG: @@ -0,0 +1,1 @@
--- a/tests/test-conflict.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-conflict.t Sun Nov 27 20:44:52 2016 -0500 @@ -55,7 +55,7 @@ ======= 4 5 - >>>>>>> merge rev: c0c68e4fe667 - test: branch1 + >>>>>>> merge rev: c0c68e4fe667 - test: branch1 Hop we are done. $ hg status
--- a/tests/test-contrib-perf.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-contrib-perf.t Sun Nov 27 20:44:52 2016 -0500 @@ -50,6 +50,7 @@ perfancestorset (no help text available) perfannotate (no help text available) + perfbdiff benchmark a bdiff between revisions perfbranchmap benchmark the update of a branchmap perfcca (no help text available) @@ -94,6 +95,8 @@ (no help text available) perfrawfiles (no help text available) perfrevlog Benchmark reading a series of revisions from a revlog. + perfrevlogchunks + Benchmark operations on revlog chunks. perfrevlogrevision Benchmark obtaining a revlog revision. perfrevrange (no help text available) @@ -112,6 +115,8 @@ $ hg perfancestors $ hg perfancestorset 2 $ hg perfannotate a + $ hg perfbdiff -c 1 + $ hg perfbdiff --alldata 1 $ hg perfbranchmap $ hg perfcca $ hg perfchangegroupchangelog @@ -141,6 +146,7 @@ $ hg perfrawfiles 2 $ hg perfrevlog .hg/store/data/a.i $ hg perfrevlogrevision -m 0 + $ hg perfrevlogchunks -c $ hg perfrevrange $ hg perfrevset 'all()' $ hg perfstartup
--- a/tests/test-convert-darcs.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-convert-darcs.t Sun Nov 27 20:44:52 2016 -0500 @@ -8,7 +8,7 @@ $ mkdir darcs-repo $ cd darcs-repo - $ darcs init + $ darcs init -q $ echo a > a $ darcs record -a -l -m p0 Finished recording patch 'p0' @@ -43,6 +43,7 @@ Backing up ./a(*) (glob) We have conflicts in the following files: ./a + (?) $ sleep 1 $ echo e > a $ echo f > f @@ -54,13 +55,13 @@ test file and directory move - $ darcs mv f ff + $ darcs mv -q f ff Test remove + move - $ darcs remove dir/d2 + $ darcs remove -q dir/d2 $ rm dir/d2 - $ darcs mv dir dir2 + $ darcs mv -q dir dir2 $ darcs record -a -l -m p3 Finished recording patch 'p3'
--- a/tests/test-debugcommands.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-debugcommands.t Sun Nov 27 20:44:52 2016 -0500 @@ -22,6 +22,11 @@ full : 44 (100.00%) deltas : 0 ( 0.00%) + chunks : 1 + 0x75 (u) : 1 (100.00%) + chunks size : 44 + 0x75 (u) : 44 (100.00%) + avg chain length : 0 max chain length : 0 compression ratio : 0
--- a/tests/test-devel-warnings.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-devel-warnings.t Sun Nov 27 20:44:52 2016 -0500 @@ -92,6 +92,7 @@ */mercurial/dispatch.py:* in dispatch (glob) */mercurial/dispatch.py:* in _runcatch (glob) */mercurial/dispatch.py:* in callcatch (glob) + */mercurial/scmutil.py* in callcatch (glob) */mercurial/dispatch.py:* in _runcatchfunc (glob) */mercurial/dispatch.py:* in _dispatch (glob) */mercurial/dispatch.py:* in runcommand (glob) @@ -127,6 +128,7 @@ */mercurial/dispatch.py:* in dispatch (glob) */mercurial/dispatch.py:* in _runcatch (glob) */mercurial/dispatch.py:* in callcatch (glob) + */mercurial/scmutil.py* in callcatch (glob) */mercurial/dispatch.py:* in _runcatchfunc (glob) */mercurial/dispatch.py:* in _dispatch (glob) */mercurial/dispatch.py:* in runcommand (glob) @@ -150,6 +152,7 @@ */mercurial/dispatch.py:* in dispatch (glob) */mercurial/dispatch.py:* in _runcatch (glob) */mercurial/dispatch.py:* in callcatch (glob) + */mercurial/scmutil.py* in callcatch (glob) */mercurial/dispatch.py:* in _runcatchfunc (glob) */mercurial/dispatch.py:* in _dispatch (glob) */mercurial/dispatch.py:* in runcommand (glob)
--- a/tests/test-extension.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-extension.t Sun Nov 27 20:44:52 2016 -0500 @@ -251,10 +251,10 @@ #if demandimport absimport -Examine whether module loading is delayed until actual refering, even +Examine whether module loading is delayed until actual referring, even though module is imported with "absolute_import" feature. -Files below in each packages are used for descirbed purpose: +Files below in each packages are used for described purpose: - "called": examine whether "from MODULE import ATTR" works correctly - "unused": examine whether loading is delayed correctly @@ -1045,6 +1045,61 @@ $ hg help patchbomb patchbomb extension - command to send changesets as (a series of) patch emails + The series is started off with a "[PATCH 0 of N]" introduction, which + describes the series as a whole. + + Each patch email has a Subject line of "[PATCH M of N] ...", using the first + line of the changeset description as the subject text. The message contains + two or three body parts: + + - The changeset description. + - [Optional] The result of running diffstat on the patch. + - The patch itself, as generated by 'hg export'. + + Each message refers to the first in the series using the In-Reply-To and + References headers, so they will show up as a sequence in threaded mail and + news readers, and in mail archives. + + To configure other defaults, add a section like this to your configuration + file: + + [email] + from = My Name <my@email> + to = recipient1, recipient2, ... + cc = cc1, cc2, ... + bcc = bcc1, bcc2, ... + reply-to = address1, address2, ... + + Use "[patchbomb]" as configuration section name if you need to override global + "[email]" address settings. + + Then you can use the 'hg email' command to mail a series of changesets as a + patchbomb. + + You can also either configure the method option in the email section to be a + sendmail compatible mailer or fill out the [smtp] section so that the + patchbomb extension can automatically send patchbombs directly from the + commandline. See the [email] and [smtp] sections in hgrc(5) for details. + + By default, 'hg email' will prompt for a "To" or "CC" header if you do not + supply one via configuration or the command line. You can override this to + never prompt by configuring an empty value: + + [email] + cc = + + You can control the default inclusion of an introduction message with the + "patchbomb.intro" configuration option. The configuration is always + overwritten by command line flags like --intro and --desc: + + [patchbomb] + intro=auto # include introduction message if more than 1 patch (default) + intro=never # never include an introduction message + intro=always # always include an introduction message + + You can set patchbomb to always ask for confirmation by setting + "patchbomb.confirm" to true. + (use 'hg help extensions' for information on enabling extensions) @@ -1455,48 +1510,6 @@ $ cd .. -Test compatibility with extension commands that don't use @command (issue5137) - - $ hg init deprecated - $ cd deprecated - - $ cat <<EOF > deprecatedcmd.py - > def deprecatedcmd(repo, ui): - > pass - > cmdtable = { - > 'deprecatedcmd': (deprecatedcmd, [], ''), - > } - > EOF - $ cat <<EOF > .hg/hgrc - > [extensions] - > deprecatedcmd = `pwd`/deprecatedcmd.py - > mq = ! - > hgext.mq = ! - > hgext/mq = ! - > [alias] - > deprecatedalias = deprecatedcmd - > EOF - - $ hg deprecatedcmd - devel-warn: missing attribute 'norepo', use @command decorator to register 'deprecatedcmd' - (compatibility will be dropped after Mercurial-3.8, update your code.) at: * (glob) - - $ hg deprecatedalias - devel-warn: missing attribute 'norepo', use @command decorator to register 'deprecatedalias' - (compatibility will be dropped after Mercurial-3.8, update your code.) at: * (glob) - - no warning unless command is executed: - - $ hg paths - - but mq iterates over command table: - - $ hg --config extensions.mq= paths - devel-warn: missing attribute 'norepo', use @command decorator to register 'deprecatedcmd' - (compatibility will be dropped after Mercurial-3.8, update your code.) at: * (glob) - - $ cd .. - Test synopsis and docstring extending $ hg init exthelp
--- a/tests/test-filecache.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-filecache.py Sun Nov 27 20:44:52 2016 -0500 @@ -197,7 +197,7 @@ repetition = 3 # repeat changing via checkambigatclosing, to examine whether - # st_mtime is advanced multiple times as expecetd + # st_mtime is advanced multiple times as expected for i in xrange(repetition): # explicit closing fp = scmutil.checkambigatclosing(open(filename, 'a')) @@ -214,7 +214,7 @@ continue # st_mtime should be advanced "repetition * 2" times, because - # all changes occured at same time (in sec) + # all changes occurred at same time (in sec) expected = (oldstat.st_mtime + repetition * 2) & 0x7fffffff if newstat.st_mtime != expected: print("'newstat.st_mtime %s is not %s (as %s + %s * 2)" %
--- a/tests/test-generaldelta.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-generaldelta.t Sun Nov 27 20:44:52 2016 -0500 @@ -74,7 +74,7 @@ $ cd .. Test "usegeneraldelta" config -(repo are general delta, but incoming bundle are not re-deltified) +(repo are general delta, but incoming bundle are not re-deltafied) delta coming from the server base delta server are not recompressed. (also include the aggressive version for comparison)
--- a/tests/test-graft.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-graft.t Sun Nov 27 20:44:52 2016 -0500 @@ -456,7 +456,7 @@ c ======= b - >>>>>>> graft: 5d205f8b35b6 - bar: 1 + >>>>>>> graft: 5d205f8b35b6 - bar: 1 $ echo b > a $ hg resolve -m a (no more unresolved files)
--- a/tests/test-hardlinks.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-hardlinks.t Sun Nov 27 20:44:52 2016 -0500 @@ -211,6 +211,8 @@ 2 r4/.hg/00changelog.i 2 r4/.hg/branch 2 r4/.hg/cache/branch2-served + 2 r4/.hg/cache/checkisexec + 2 r4/.hg/cache/checknoexec 2 r4/.hg/cache/rbc-names-v1 2 r4/.hg/cache/rbc-revs-v1 2 r4/.hg/dirstate @@ -247,6 +249,8 @@ 2 r4/.hg/00changelog.i 1 r4/.hg/branch 2 r4/.hg/cache/branch2-served + 2 r4/.hg/cache/checkisexec + 2 r4/.hg/cache/checknoexec 2 r4/.hg/cache/rbc-names-v1 2 r4/.hg/cache/rbc-revs-v1 1 r4/.hg/dirstate
--- a/tests/test-help.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-help.t Sun Nov 27 20:44:52 2016 -0500 @@ -241,7 +241,6 @@ enabled extensions: - chgserver command server extension for cHg (EXPERIMENTAL) (?) children command to display child changesets (DEPRECATED) rebase command to move sets of revisions to a different ancestor
--- a/tests/test-hgrc.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-hgrc.t Sun Nov 27 20:44:52 2016 -0500 @@ -46,7 +46,6 @@ default = $TESTTMP/foo%bar (glob) $ hg showconfig bundle.mainreporoot=$TESTTMP/foobar (glob) - extensions.chgserver= (?) paths.default=$TESTTMP/foo%bar (glob) $ cd .. @@ -81,7 +80,6 @@ $ hg showconfig --config ui.verbose=True --quiet bundle.mainreporoot=$TESTTMP - extensions.chgserver= (?) ui.verbose=False ui.debug=False ui.quiet=True @@ -113,7 +111,6 @@ $ hg showconfig bundle.mainreporoot=$TESTTMP - extensions.chgserver= (?) ui.username=$FAKEUSER $ unset FAKEUSER @@ -159,7 +156,6 @@ $TESTTMP/hgrc:13: alias.log=log -g repo: bundle.mainreporoot=$TESTTMP $TESTTMP/hgrc:11: defaults.identify=-n - --config: extensions.chgserver= (?) $TESTTMP/hgrc:2: ui.debug=true $TESTTMP/hgrc:3: ui.fallbackencoding=ASCII $TESTTMP/hgrc:4: ui.quiet=true @@ -175,7 +171,6 @@ $ hg showconfig --config ui.traceback=True --debug read config from: $TESTTMP/hgrc repo: bundle.mainreporoot=$TESTTMP - --config: extensions.chgserver= (?) --config: ui.traceback=True --verbose: ui.verbose=False --debug: ui.debug=True @@ -199,7 +194,6 @@ read config from: $TESTTMP/hgrc repo: bundle.mainreporoot=$TESTTMP $TESTTMP/hgrc:15: extensions.plain=./plain.py - --config: extensions.chgserver= (?) --config: ui.traceback=True --verbose: ui.verbose=False --debug: ui.debug=True @@ -210,7 +204,6 @@ read config from: $TESTTMP/hgrc repo: bundle.mainreporoot=$TESTTMP $TESTTMP/hgrc:15: extensions.plain=./plain.py - --config: extensions.chgserver= (?) --config: ui.traceback=True --verbose: ui.verbose=False --debug: ui.debug=True @@ -221,7 +214,6 @@ read config from: $TESTTMP/hgrc repo: bundle.mainreporoot=$TESTTMP $TESTTMP/hgrc:15: extensions.plain=./plain.py - --config: extensions.chgserver= (?) --config: ui.traceback=True --verbose: ui.verbose=False --debug: ui.debug=True
--- a/tests/test-https.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-https.t Sun Nov 27 20:44:52 2016 -0500 @@ -379,7 +379,7 @@ Fingerprints -- works without cacerts (hostkeyfingerprints) +- works without cacerts (hostfingerprints) $ hg -R copy-pull id https://localhost:$HGPORT/ --insecure --config hostfingerprints.localhost=ec:d8:7c:d6:b3:86:d0:4f:c1:b8:b4:1c:9d:8f:5e:16:8e:ef:1c:03 warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?) 5fed3813f7f5
--- a/tests/test-import.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-import.t Sun Nov 27 20:44:52 2016 -0500 @@ -431,10 +431,10 @@ parent: 0 Test that "hg rollback" doesn't restore dirstate to one at the -beginning of the rollbacked transaction in not-"parent-gone" case. +beginning of the rolled back transaction in not-"parent-gone" case. invoking pretxncommit hook will cause marking '.hg/dirstate' as a file -to be restored at rollbacking, after DirstateTransactionPlan (see wiki +to be restored when rolling back, after DirstateTransactionPlan (see wiki page for detail). $ hg --cwd b branch -q foobar @@ -451,7 +451,7 @@ $ hg --cwd b update -q -C 0 $ hg --cwd b --config extensions.strip= strip -q 1 -Test visibility of in-memory distate changes inside transaction to +Test visibility of in-memory dirstate changes inside transaction to external process $ echo foo > a/foo
--- a/tests/test-install.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-install.t Sun Nov 27 20:44:52 2016 -0500 @@ -11,6 +11,8 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob) @@ -21,6 +23,8 @@ $ hg debuginstall -Tjson | sed 's|\\\\|\\|g' [ { + "compengines": ["bz2", "bz2truncated", "none", "zlib"*], (glob) + "compenginesavail": ["bz2", "bz2truncated", "none", "zlib"*], (glob) "defaulttemplate": "*mercurial?templates?map-cmdline.default", (glob) "defaulttemplateerror": null, "defaulttemplatenotfound": "default", @@ -58,6 +62,8 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob) @@ -85,6 +91,8 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob)
--- a/tests/test-keyword.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-keyword.t Sun Nov 27 20:44:52 2016 -0500 @@ -1117,7 +1117,7 @@ bar ======= foo - >>>>>>> merge rev: 85d2d2d732a5 - test: simplemerge + >>>>>>> merge rev: 85d2d2d732a5 - test: simplemerge resolve to local, m must contain hash of last change (local parent)
--- a/tests/test-largefiles-wireproto.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-largefiles-wireproto.t Sun Nov 27 20:44:52 2016 -0500 @@ -388,7 +388,7 @@ $ killdaemons.py -largefiles should not ask for password again after succesfull authorization +largefiles should not ask for password again after successful authorization $ hg init credentialmain $ cd credentialmain
--- a/tests/test-merge-changedelete.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge-changedelete.t Sun Nov 27 20:44:52 2016 -0500 @@ -107,7 +107,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge: @@ -171,7 +171,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge with bad input: @@ -247,7 +247,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge with not enough input: @@ -310,7 +310,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Choose local versions of files @@ -635,7 +635,7 @@ ||||||| base ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Exercise transitions between local, other, fail and prompt, and make sure the dirstate stays consistent. (Compare with each other and to the above
--- a/tests/test-merge-force.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge-force.t Sun Nov 27 20:44:52 2016 -0500 @@ -379,7 +379,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content1_content4-untracked content2 @@ -409,7 +409,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content2_content4-untracked content2 @@ -439,7 +439,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content3_content3-untracked content2 @@ -451,7 +451,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content3_content4-untracked content2 @@ -481,7 +481,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_missing_content4-untracked content2 @@ -564,7 +564,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content2_content4-untracked content2 @@ -587,7 +587,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content3_content3-untracked content2 @@ -598,7 +598,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content3_content4-untracked content2 @@ -621,7 +621,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_missing_content4-untracked <<<<<<< working copy: 0447570f1af6 - test: local @@ -629,7 +629,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_missing_missing-tracked content2
--- a/tests/test-merge-revert2.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge-revert2.t Sun Nov 27 20:44:52 2016 -0500 @@ -56,7 +56,7 @@ @@ -1,3 +1,7 @@ added file1 another line of text - +<<<<<<< working copy: c3fa057dd86f - test: added file1 and file2 + +<<<<<<< working copy: c3fa057dd86f - test: added file1 and file2 +changed file1 different +======= changed file1
--- a/tests/test-merge-tools.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge-tools.t Sun Nov 27 20:44:52 2016 -0500 @@ -75,11 +75,11 @@ [1] $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 ======= revision 2 - >>>>>>> merge rev: 0185f4e0cf02 - test: revision 2 + >>>>>>> merge rev: 0185f4e0cf02 - test: revision 2 space # hg stat M f @@ -935,7 +935,7 @@ # hg update -C 1 $ hg merge -r 4 --config merge-tools.true.premerge=keep merging f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ======= @@ -948,7 +948,7 @@ (branch merge, don't forget to commit) $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ======= @@ -969,7 +969,7 @@ # hg update -C 1 $ hg merge -r 4 --config merge-tools.true.premerge=keep-merge3 merging f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ||||||| base @@ -985,7 +985,7 @@ (branch merge, don't forget to commit) $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ||||||| base @@ -1209,3 +1209,15 @@ [1] #endif + +Verify naming of temporary files and that extension is preserved: + + $ hg update -q -C 1 + $ hg mv f f.txt + $ hg ci -qm "f.txt" + $ hg update -q -C 2 + $ hg merge -y -r tip --tool echo --config merge-tools.echo.args='$base $local $other $output' + merging f and f.txt to f.txt + */f~base.?????? $TESTTMP/f.txt.orig */f~other.??????.txt $TESTTMP/f.txt (glob) + 0 files updated, 1 files merged, 0 files removed, 0 files unresolved + (branch merge, don't forget to commit)
--- a/tests/test-merge-types.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge-types.t Sun Nov 27 20:44:52 2016 -0500 @@ -367,14 +367,14 @@ 2 ======= 1 - >>>>>>> merge rev: 2e60aa20b912 - test: 1 + >>>>>>> merge rev: 2e60aa20b912 - test: 1 $ tellmeabout b b is a plain file with content: <<<<<<< working copy: 0c617753b41b - test: 2 2 ======= 1 - >>>>>>> merge rev: 2e60aa20b912 - test: 1 + >>>>>>> merge rev: 2e60aa20b912 - test: 1 $ tellmeabout c c is a plain file with content: x @@ -418,14 +418,14 @@ [1] $ tellmeabout a a is a plain file with content: - <<<<<<< working copy: 2e60aa20b912 - test: 1 + <<<<<<< working copy: 2e60aa20b912 - test: 1 1 ======= 2 >>>>>>> merge rev: 0c617753b41b - test: 2 $ tellmeabout b b is an executable file with content: - <<<<<<< working copy: 2e60aa20b912 - test: 1 + <<<<<<< working copy: 2e60aa20b912 - test: 1 1 ======= 2
--- a/tests/test-merge7.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-merge7.t Sun Nov 27 20:44:52 2016 -0500 @@ -99,7 +99,7 @@ $ cat test.txt one - <<<<<<< working copy: 50c3a7e29886 - test: Merge 1 + <<<<<<< working copy: 50c3a7e29886 - test: Merge 1 two-point-five ======= two-point-one
--- a/tests/test-mq-qrefresh-replace-log-message.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-mq-qrefresh-replace-log-message.t Sun Nov 27 20:44:52 2016 -0500 @@ -199,7 +199,7 @@ test saving last-message.txt -Test visibility of in-memory distate changes outside transaction to +Test visibility of in-memory dirstate changes outside transaction to external process $ cat > $TESTTMP/checkvisibility.sh <<EOF
--- a/tests/test-obsolete.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-obsolete.t Sun Nov 27 20:44:52 2016 -0500 @@ -1211,7 +1211,7 @@ 2 1ab51af8f9b41ef8c7f6f3312d4706d870b1fb74 29346082e4a9e27042b62d2da0e2de211c027621 0 \(.*\) {'user': 'test'} (re) 3 4715cf767440ed891755448016c2b8cf70760c30 7ae79c5d60f049c7b0dd02f5f25b9d60aaf7b36d 0 \(.*\) {'user': 'test'} (re) $ hg debugobsolete --delete 1 --delete 3 - deleted 2 obsolescense markers + deleted 2 obsolescence markers $ hg debugobsolete cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b f9bd49731b0b175e42992a3c8fa6c678b2bc11f1 0 \(.*\) {'user': 'test'} (re) 1ab51af8f9b41ef8c7f6f3312d4706d870b1fb74 29346082e4a9e27042b62d2da0e2de211c027621 0 \(.*\) {'user': 'test'} (re)
--- a/tests/test-profile.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-profile.t Sun Nov 27 20:44:52 2016 -0500 @@ -8,25 +8,27 @@ test --profile - $ hg --profile st 2>../out + $ prof='hg --config profiling.type=ls --profile' + + $ $prof st 2>../out $ grep CallCount ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=../out st + $ $prof --config profiling.output=../out st $ grep CallCount ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=blackbox --config extensions.blackbox= st + $ $prof --config profiling.output=blackbox --config extensions.blackbox= st $ grep CallCount .hg/blackbox.log > /dev/null || cat .hg/blackbox.log - $ hg --profile --config profiling.format=text st 2>../out + $ $prof --config profiling.format=text st 2>../out $ grep CallCount ../out > /dev/null || cat ../out $ echo "[profiling]" >> $HGRCPATH $ echo "format=kcachegrind" >> $HGRCPATH - $ hg --profile st 2>../out + $ $prof st 2>../out $ grep 'events: Ticks' ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=../out st + $ $prof --config profiling.output=../out st $ grep 'events: Ticks' ../out > /dev/null || cat ../out #endif @@ -35,7 +37,7 @@ Profiling of HTTP requests works - $ hg --profile --config profiling.format=text --config profiling.output=../profile.log serve -d -p $HGPORT --pid-file ../hg.pid -A ../access.log + $ $prof --config profiling.format=text --config profiling.output=../profile.log serve -d -p $HGPORT --pid-file ../hg.pid -A ../access.log $ cat ../hg.pid >> $DAEMON_PIDS $ hg -q clone -U http://localhost:$HGPORT ../clone @@ -45,4 +47,49 @@ #endif +Install an extension that can sleep and guarantee a profiler has time to run + + $ cat >> sleepext.py << EOF + > import time + > from mercurial import cmdutil, commands + > cmdtable = {} + > command = cmdutil.command(cmdtable) + > @command('sleep', [], 'hg sleep') + > def sleep(ui, *args, **kwargs): + > time.sleep(0.1) + > EOF + + $ cat >> $HGRCPATH << EOF + > [extensions] + > sleep = `pwd`/sleepext.py + > EOF + +statistical profiler works + + $ hg --profile sleep 2>../out + $ grep Sample ../out + Sample count: \d+ (re) + +Various statprof formatters work + + $ hg --profile --config profiling.statformat=byline sleep 2>../out + $ head -n 1 ../out + % cumulative self + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=bymethod sleep 2>../out + $ head -n 1 ../out + % cumulative self + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=hotpath sleep 2>../out + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=json sleep 2>../out + $ cat ../out + \[\[\d+.* (re) + $ cd ..
--- a/tests/test-pull-update.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-pull-update.t Sun Nov 27 20:44:52 2016 -0500 @@ -130,7 +130,7 @@ Test that updating deactivates current active bookmark, if the destination of the update is explicitly specified, and it doesn't -match with the name of any exsiting bookmarks. +match with the name of any existing bookmarks. $ cd ../t $ hg bookmark -d active-after-pull
--- a/tests/test-push-hook-lock.t Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,50 +0,0 @@ - $ hg init 1 - - $ echo '[ui]' >> 1/.hg/hgrc - $ echo 'timeout = 10' >> 1/.hg/hgrc - - $ echo foo > 1/foo - $ hg --cwd 1 ci -A -m foo - adding foo - - $ hg clone 1 2 - updating to branch default - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - - $ hg clone 2 3 - updating to branch default - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - - $ cat <<EOF > $TESTTMP/debuglocks-pretxn-hook.sh - > hg debuglocks - > true - > EOF - $ echo '[hooks]' >> 2/.hg/hgrc - $ echo "pretxnchangegroup.a = sh $TESTTMP/debuglocks-pretxn-hook.sh" >> 2/.hg/hgrc - $ echo 'changegroup.push = hg push -qf ../1' >> 2/.hg/hgrc - - $ echo bar >> 3/foo - $ hg --cwd 3 ci -m bar - - $ hg --cwd 3 push ../2 --config devel.legacy.exchange=bundle1 - pushing to ../2 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - lock: user *, process * (*s) (glob) - wlock: free - - $ hg --cwd 1 --config extensions.strip= strip tip -q - $ hg --cwd 2 --config extensions.strip= strip tip -q - $ hg --cwd 3 push ../2 # bundle2+ - pushing to ../2 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - lock: user *, process * (*s) (glob) - wlock: user *, process * (*s) (glob) -
--- a/tests/test-push-r.t Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,149 +0,0 @@ - $ hg init test - $ cd test - $ hg unbundle "$TESTDIR/bundles/remote.hg" - adding changesets - adding manifests - adding file changes - added 9 changesets with 7 changes to 4 files (+1 heads) - (run 'hg heads' to see heads, 'hg merge' to merge) - $ hg up tip - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ cd .. - - $ for i in 0 1 2 3 4 5 6 7 8; do - > echo - > mkdir test-"$i" - > hg --cwd test-"$i" init - > hg -R test push -r "$i" test-"$i" - > cd test-"$i" - > hg verify - > cd .. - > done - - pushing to test-0 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 1 changesets, 1 total revisions - - pushing to test-1 - searching for changes - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 2 changesets, 2 total revisions - - pushing to test-2 - searching for changes - adding changesets - adding manifests - adding file changes - added 3 changesets with 3 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 3 changesets, 3 total revisions - - pushing to test-3 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 4 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 4 changesets, 4 total revisions - - pushing to test-4 - searching for changes - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 2 changesets, 2 total revisions - - pushing to test-5 - searching for changes - adding changesets - adding manifests - adding file changes - added 3 changesets with 3 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 3 changesets, 3 total revisions - - pushing to test-6 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 5 changes to 2 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 2 files, 4 changesets, 5 total revisions - - pushing to test-7 - searching for changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 6 changes to 3 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 3 files, 5 changesets, 6 total revisions - - pushing to test-8 - searching for changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 5 changes to 2 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 2 files, 5 changesets, 5 total revisions - - $ cd test-8 - - $ hg pull ../test-7 - pulling from ../test-7 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 2 changes to 3 files (+1 heads) - (run 'hg heads' to see heads, 'hg merge' to merge) - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 4 files, 9 changesets, 7 total revisions - - $ cd ..
--- a/tests/test-push-validation.t Mon Nov 14 22:43:25 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,92 +0,0 @@ - $ hg init test - $ cd test - - $ cat > .hg/hgrc <<EOF - > [server] - > validate=1 - > EOF - - $ echo alpha > alpha - $ echo beta > beta - $ hg addr - adding alpha - adding beta - $ hg ci -m 1 - - $ cd .. - $ hg clone test test-clone - updating to branch default - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved - -Test spurious filelog entries: - - $ cd test-clone - $ echo blah >> beta - $ cp .hg/store/data/beta.i tmp1 - $ hg ci -m 2 - $ cp .hg/store/data/beta.i tmp2 - $ hg -q rollback - $ mv tmp2 .hg/store/data/beta.i - $ echo blah >> beta - $ hg ci -m '2 (corrupt)' - -Expected to fail: - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - beta@1: dddc47b3ba30 not in manifests - 2 files, 2 changesets, 4 total revisions - 1 integrity errors encountered! - (first damaged changeset appears to be 1) - [1] - - $ hg push - pushing to $TESTTMP/test (glob) - searching for changes - adding changesets - adding manifests - adding file changes - transaction abort! - rollback completed - abort: received spurious file revlog entry - [255] - - $ hg -q rollback - $ mv tmp1 .hg/store/data/beta.i - $ echo beta > beta - -Test missing filelog entries: - - $ cp .hg/store/data/beta.i tmp - $ echo blah >> beta - $ hg ci -m '2 (corrupt)' - $ mv tmp .hg/store/data/beta.i - -Expected to fail: - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - beta@1: manifest refers to unknown revision dddc47b3ba30 - 2 files, 2 changesets, 2 total revisions - 1 integrity errors encountered! - (first damaged changeset appears to be 1) - [1] - - $ hg push - pushing to $TESTTMP/test (glob) - searching for changes - adding changesets - adding manifests - adding file changes - transaction abort! - rollback completed - abort: missing file data for beta:dddc47b3ba30e54484720ce0f4f768a0f4b6efb9 - run hg verify - [255] - - $ cd ..
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test-push.t Sun Nov 27 20:44:52 2016 -0500 @@ -0,0 +1,299 @@ +================================== +Basic testing for the push command +================================== + +Testing of the '--rev' flag +=========================== + + $ hg init test-revflag + $ hg -R test-revflag unbundle "$TESTDIR/bundles/remote.hg" + adding changesets + adding manifests + adding file changes + added 9 changesets with 7 changes to 4 files (+1 heads) + (run 'hg heads' to see heads, 'hg merge' to merge) + + $ for i in 0 1 2 3 4 5 6 7 8; do + > echo + > hg init test-revflag-"$i" + > hg -R test-revflag push -r "$i" test-revflag-"$i" + > hg -R test-revflag-"$i" verify + > done + + pushing to test-revflag-0 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 1 changesets, 1 total revisions + + pushing to test-revflag-1 + searching for changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 2 changesets, 2 total revisions + + pushing to test-revflag-2 + searching for changes + adding changesets + adding manifests + adding file changes + added 3 changesets with 3 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 3 changesets, 3 total revisions + + pushing to test-revflag-3 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 4 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 4 changesets, 4 total revisions + + pushing to test-revflag-4 + searching for changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 2 changesets, 2 total revisions + + pushing to test-revflag-5 + searching for changes + adding changesets + adding manifests + adding file changes + added 3 changesets with 3 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 3 changesets, 3 total revisions + + pushing to test-revflag-6 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 5 changes to 2 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 2 files, 4 changesets, 5 total revisions + + pushing to test-revflag-7 + searching for changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 6 changes to 3 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 3 files, 5 changesets, 6 total revisions + + pushing to test-revflag-8 + searching for changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 5 changes to 2 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 2 files, 5 changesets, 5 total revisions + + $ cd test-revflag-8 + + $ hg pull ../test-revflag-7 + pulling from ../test-revflag-7 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 2 changes to 3 files (+1 heads) + (run 'hg heads' to see heads, 'hg merge' to merge) + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 4 files, 9 changesets, 7 total revisions + + $ cd .. + +Test server side validation during push +======================================= + + $ hg init test-validation + $ cd test-validation + + $ cat > .hg/hgrc <<EOF + > [server] + > validate=1 + > EOF + + $ echo alpha > alpha + $ echo beta > beta + $ hg addr + adding alpha + adding beta + $ hg ci -m 1 + + $ cd .. + $ hg clone test-validation test-validation-clone + updating to branch default + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + +Test spurious filelog entries: + + $ cd test-validation-clone + $ echo blah >> beta + $ cp .hg/store/data/beta.i tmp1 + $ hg ci -m 2 + $ cp .hg/store/data/beta.i tmp2 + $ hg -q rollback + $ mv tmp2 .hg/store/data/beta.i + $ echo blah >> beta + $ hg ci -m '2 (corrupt)' + +Expected to fail: + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + beta@1: dddc47b3ba30 not in manifests + 2 files, 2 changesets, 4 total revisions + 1 integrity errors encountered! + (first damaged changeset appears to be 1) + [1] + + $ hg push + pushing to $TESTTMP/test-validation (glob) + searching for changes + adding changesets + adding manifests + adding file changes + transaction abort! + rollback completed + abort: received spurious file revlog entry + [255] + + $ hg -q rollback + $ mv tmp1 .hg/store/data/beta.i + $ echo beta > beta + +Test missing filelog entries: + + $ cp .hg/store/data/beta.i tmp + $ echo blah >> beta + $ hg ci -m '2 (corrupt)' + $ mv tmp .hg/store/data/beta.i + +Expected to fail: + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + beta@1: manifest refers to unknown revision dddc47b3ba30 + 2 files, 2 changesets, 2 total revisions + 1 integrity errors encountered! + (first damaged changeset appears to be 1) + [1] + + $ hg push + pushing to $TESTTMP/test-validation (glob) + searching for changes + adding changesets + adding manifests + adding file changes + transaction abort! + rollback completed + abort: missing file data for beta:dddc47b3ba30e54484720ce0f4f768a0f4b6efb9 - run hg verify + [255] + + $ cd .. + +Test push hook locking +===================== + + $ hg init 1 + + $ echo '[ui]' >> 1/.hg/hgrc + $ echo 'timeout = 10' >> 1/.hg/hgrc + + $ echo foo > 1/foo + $ hg --cwd 1 ci -A -m foo + adding foo + + $ hg clone 1 2 + updating to branch default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + + $ hg clone 2 3 + updating to branch default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + + $ cat <<EOF > $TESTTMP/debuglocks-pretxn-hook.sh + > hg debuglocks + > true + > EOF + $ echo '[hooks]' >> 2/.hg/hgrc + $ echo "pretxnchangegroup.a = sh $TESTTMP/debuglocks-pretxn-hook.sh" >> 2/.hg/hgrc + $ echo 'changegroup.push = hg push -qf ../1' >> 2/.hg/hgrc + + $ echo bar >> 3/foo + $ hg --cwd 3 ci -m bar + + $ hg --cwd 3 push ../2 --config devel.legacy.exchange=bundle1 + pushing to ../2 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + lock: user *, process * (*s) (glob) + wlock: free + + $ hg --cwd 1 --config extensions.strip= strip tip -q + $ hg --cwd 2 --config extensions.strip= strip tip -q + $ hg --cwd 3 push ../2 # bundle2+ + pushing to ../2 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + lock: user *, process * (*s) (glob) + wlock: user *, process * (*s) (glob) +
--- a/tests/test-qrecord.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-qrecord.t Sun Nov 27 20:44:52 2016 -0500 @@ -9,6 +9,9 @@ record extension - commands to interactively select changes for commit/qrefresh (DEPRECATED) + The feature provided by this extension has been moved into core Mercurial as + 'hg commit --interactive'. + (use 'hg help extensions' for information on enabling extensions) help qrecord (no record)
--- a/tests/test-rebase-collapse.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-rebase-collapse.t Sun Nov 27 20:44:52 2016 -0500 @@ -836,7 +836,7 @@ rebasing 2:b8d8db2b242d "a-dev" (tip) saved backup bundle to $TESTTMP/collapse_remember_message/.hg/strip-backup/b8d8db2b242d-f474c19a-backup.hg (glob) $ hg log - changeset: 2:12bb766dceb1 + changeset: 2:45ba1d1a8665 tag: tip user: test date: Thu Jan 01 00:00:00 1970 +0000
--- a/tests/test-rebase-conflicts.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-rebase-conflicts.t Sun Nov 27 20:44:52 2016 -0500 @@ -73,8 +73,6 @@ Try to continue without solving the conflict: $ hg rebase --continue - already rebased 3:3163e20567cc "L1" as 3e046f2ecedb - rebasing 4:46f0b057b5c0 "L2" abort: unresolved merge conflicts (see 'hg help resolve') [255] @@ -335,7 +333,7 @@ @@ -1,2 +1,6 @@ a b - +<<<<<<< dest: 328e4ab1f7cc ab - test: ab + +<<<<<<< dest: 328e4ab1f7cc ab - test: ab +======= +c +>>>>>>> source: 7bc217434fc1 - test: abc @@ -354,7 +352,7 @@ +++ b/a * (glob) @@ -1,2 +1,8 @@ a - +<<<<<<< dest: 328e4ab1f7cc ab - test: ab + +<<<<<<< dest: 328e4ab1f7cc ab - test: ab b +||||||| base +=======
--- a/tests/test-rebase-scenario-global.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-rebase-scenario-global.t Sun Nov 27 20:44:52 2016 -0500 @@ -838,7 +838,7 @@ $ cd .. -Make the repo a bit more interresting +Make the repo a bit more interesting $ hg up 1 0 files updated, 0 files merged, 2 files removed, 0 files unresolved
--- a/tests/test-rename-dir-merge.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-rename-dir-merge.t Sun Nov 27 20:44:52 2016 -0500 @@ -148,7 +148,7 @@ target ======= baz - >>>>>>> merge rev: ce36d17b18fb - test: 2 add a/c + >>>>>>> merge rev: ce36d17b18fb - test: 2 add a/c $ rm b/c.orig Remote directory rename with conflicting file added in remote target directory @@ -177,7 +177,7 @@ ? a/d ? b/c.orig $ cat b/c - <<<<<<< working copy: ce36d17b18fb - test: 2 add a/c + <<<<<<< working copy: ce36d17b18fb - test: 2 add a/c baz ======= target
--- a/tests/test-revert-interactive.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-revert-interactive.t Sun Nov 27 20:44:52 2016 -0500 @@ -46,6 +46,7 @@ > y > y > y + > y > n > n > EOF @@ -53,6 +54,7 @@ reverting folder1/g (glob) removing folder1/i (glob) reverting folder2/h (glob) + remove added file folder1/i (Yn)? y diff --git a/f b/f 2 hunks, 2 lines changed examine changes to 'f'? [Ynesfdaq?] y @@ -137,7 +139,7 @@ $ ls folder1/ g -Test that a noop revert doesn't do an unecessary backup +Test that a noop revert doesn't do an unnecessary backup $ (echo y; echo n) | hg revert -i -r 2 folder1/g diff --git a/folder1/g b/folder1/g 1 hunks, 1 lines changed @@ -174,6 +176,7 @@ $ hg update -C 6 3 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg revert -i -r 2 --all -- << EOF + > n > y > y > y @@ -186,6 +189,7 @@ reverting folder1/g (glob) removing folder1/i (glob) reverting folder2/h (glob) + remove added file folder1/i (Yn)? n diff --git a/f b/f 2 hunks, 2 lines changed examine changes to 'f'? [Ynesfdaq?] y @@ -258,7 +262,6 @@ $ hg st M f M folder1/g - R folder1/i $ hg revert --interactive f << EOF > y > y @@ -290,7 +293,6 @@ $ hg st M f M folder1/g - R folder1/i ? f.orig $ cat f a @@ -307,7 +309,7 @@ 5 $ rm f.orig $ hg update -C . - 3 files updated, 0 files merged, 0 files removed, 0 files unresolved + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved Check editing files newly added by a revert @@ -424,14 +426,14 @@ > n > EOF forgetting newfile - forget added file newfile (yn)? n + forget added file newfile (Yn)? n $ hg status A newfile $ hg revert -i <<EOF > y > EOF forgetting newfile - forget added file newfile (yn)? y + forget added file newfile (Yn)? y $ hg status ? newfile
--- a/tests/test-revset.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-revset.t Sun Nov 27 20:44:52 2016 -0500 @@ -1079,7 +1079,7 @@ 8 9 -Test opreand of '%' is optimized recursively (issue4670) +Test operand of '%' is optimized recursively (issue4670) $ try --optimize '8:9-8%' (onlypost @@ -3085,7 +3085,7 @@ [255] test scope of alias expansion: 'universe' is expanded prior to 'shadowall(0)', -but 'all()' should never be substituded to '0()'. +but 'all()' should never be substituted to '0()'. $ echo 'universe = all()' >> .hg/hgrc $ echo 'shadowall(all) = all and universe' >> .hg/hgrc
--- a/tests/test-shelve.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-shelve.t Sun Nov 27 20:44:52 2016 -0500 @@ -332,7 +332,7 @@ +++ b/a/a @@ -1,2 +1,6 @@ a - +<<<<<<< dest: * - shelve: pending changes temporary commit (glob) + +<<<<<<< dest: * - shelve: pending changes temporary commit (glob) c +======= +a @@ -759,7 +759,7 @@ M f ? f.orig $ cat f - <<<<<<< dest: 5f6b880e719b - shelve: pending changes temporary commit + <<<<<<< dest: 5f6b880e719b - shelve: pending changes temporary commit g ======= f @@ -804,7 +804,7 @@ M f ? f.orig $ cat f - <<<<<<< dest: * - test: intermediate other change (glob) + <<<<<<< dest: * - test: intermediate other change (glob) g ======= f @@ -1493,7 +1493,7 @@ When i unshelve resulting in merge conflicts and makes saved file shelvedstate looks like in previous versions in mercurial(without restore branch information in 7th line) i -expect that after resolving conflicts and succesfully +expect that after resolving conflicts and successfully running 'shelve --continue' the branch information won't be restored and branch will be unchanged. @@ -1587,7 +1587,7 @@ default $ cd .. -Prepare unshleve with a corrupted shelvedstate +Prepare unshelve with a corrupted shelvedstate $ hg init r1 && cd r1 $ echo text1 > file && hg add file $ hg shelve @@ -1622,3 +1622,31 @@ abort: no unshelve in progress [255] $ cd .. + +Unshelve respects --keep even if user intervention is needed + $ hg init unshelvekeep + $ echo 1 > file && hg ci -Am 1 + adding file + $ echo 2 >> file + $ hg shelve + shelved as default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ echo 3 >> file && hg ci -Am 13 + $ hg shelve --list + default (1s ago) changes to: 1 + $ hg unshelve --keep + unshelving change 'default' + rebasing shelved changes + rebasing 3:1d24e58054c8 "changes to: 1" (tip) + merging file + warning: conflicts while merging file! (edit, then use 'hg resolve --mark') + unresolved conflicts (see 'hg resolve', then 'hg unshelve --continue') + [1] + $ hg resolve --mark file + (no more unresolved files) + continue: hg unshelve --continue + $ hg unshelve --continue + rebasing 3:1d24e58054c8 "changes to: 1" (tip) + unshelve of 'default' complete + $ hg shelve --list + default (1s ago) changes to: 1
--- a/tests/test-ssh.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-ssh.t Sun Nov 27 20:44:52 2016 -0500 @@ -265,8 +265,17 @@ > sys.stdout.write("KABOOM\n") > EOF - $ echo '[hooks]' >> ../remote/.hg/hgrc - $ echo "changegroup.stdout = python $TESTTMP/badhook" >> ../remote/.hg/hgrc + $ cat <<EOF > $TESTTMP/badpyhook.py + > import sys + > def hook(ui, repo, hooktype, **kwargs): + > sys.stdout.write("KABOOM IN PROCESS\n") + > EOF + + $ cat <<EOF >> ../remote/.hg/hgrc + > [hooks] + > changegroup.stdout = python $TESTTMP/badhook + > changegroup.pystdout = python:$TESTTMP/badpyhook.py:hook + > EOF $ echo r > r $ hg ci -A -m z r @@ -281,6 +290,7 @@ remote: adding file changes remote: added 1 changesets with 1 changes to 1 files remote: KABOOM + remote: KABOOM IN PROCESS $ hg -R ../remote heads changeset: 5:1383141674ec tag: tip @@ -447,6 +457,7 @@ remote: adding file changes remote: added 1 changesets with 1 changes to 1 files remote: KABOOM + remote: KABOOM IN PROCESS local stdout debug output
--- a/tests/test-subrepo.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-subrepo.t Sun Nov 27 20:44:52 2016 -0500 @@ -332,7 +332,7 @@ conflict ======= t3 - >>>>>>> other: 7af322bc1198 - test: 7 + >>>>>>> other: 7af322bc1198 - test: 7 11: remove subrepo t
--- a/tests/test-symlink-os-yes-fs-no.py Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-symlink-os-yes-fs-no.py Sun Nov 27 20:44:52 2016 -0500 @@ -35,6 +35,9 @@ def symlink_failure(src, dst): raise OSError(1, "Operation not permitted") os.symlink = symlink_failure +def islink_failure(path): + return False +os.path.islink = islink_failure # dereference links as if a Samba server has exported this to a # Windows client
--- a/tests/test-tags.t Mon Nov 14 22:43:25 2016 +0100 +++ b/tests/test-tags.t Sun Nov 27 20:44:52 2016 -0500 @@ -672,6 +672,8 @@ $ ls tagsclient/.hg/cache branch2-served + checkisexec + checklink hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 @@ -696,6 +698,8 @@ $ ls tagsclient/.hg/cache branch2-served + checkisexec + checklink hgtagsfnodes1 rbc-names-v1 rbc-revs-v1