mercurial/revlogutils/deltas.py
author Boris Feld <boris.feld@octobus.net>
Sat, 18 Aug 2018 07:27:34 +0200
changeset 39361 507f5b1dd7c8
parent 39360 5d343a24bff5
child 39362 1441eb38849f
permissions -rw-r--r--
revlogdeltas: extract _getcandidaterevs in a function The logic barely uses the object it is attached to. This is an important function that we will clean up in the coming changesets. Moving it at the top level helps us with that cleanup.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
     1
# revlogdeltas.py - Logic around delta computation for revlog
8226
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     2
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     3
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
     4
# Copyright 2018 Octobus <contact@octobus.net>
8226
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     5
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     6
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 10047
diff changeset
     7
# GNU General Public License version 2 or any later version.
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
     8
"""Helper class to compute deltas stored inside revlogs"""
8227
0a9542703300 turn some comments back into module docstrings
Martin Geisler <mg@lazybytes.net>
parents: 8226
diff changeset
     9
27361
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    10
from __future__ import absolute_import
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    11
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
    12
import heapq
27361
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    13
import struct
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    14
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    15
# import stuff from node for others to import from revlog
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    16
from ..node import (
27361
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    17
    nullrev,
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    18
)
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    19
from ..i18n import _
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    20
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    21
from .constants import (
39356
729082bb9938 revlog: split constants into a new `revlogutils.constants` module
Boris Feld <boris.feld@octobus.net>
parents: 39260
diff changeset
    22
    REVIDX_ISCENSORED,
729082bb9938 revlog: split constants into a new `revlogutils.constants` module
Boris Feld <boris.feld@octobus.net>
parents: 39260
diff changeset
    23
    REVIDX_RAWTEXT_CHANGING_FLAGS,
729082bb9938 revlog: split constants into a new `revlogutils.constants` module
Boris Feld <boris.feld@octobus.net>
parents: 39260
diff changeset
    24
)
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    25
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    26
from ..thirdparty import (
35638
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
    27
    attr,
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
    28
)
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    29
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    30
from .. import (
27361
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    31
    error,
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    32
    mdiff,
29f50344fa83 revlog: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27251
diff changeset
    33
)
10913
f2ecc5733c89 revlog: factor out _maxinline global.
Greg Ward <greg-hg@gerg.ca>
parents: 10404
diff changeset
    34
7633
08cabecfa8a8 errors: move revlog errors
Matt Mackall <mpm@selenic.com>
parents: 7365
diff changeset
    35
RevlogError = error.RevlogError
22934
8a096d4d0862 revlog: support importing censored file revision tombstones
Mike Edgar <adgar@google.com>
parents: 22785
diff changeset
    36
CensoredNodeError = error.CensoredNodeError
30745
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30744
diff changeset
    37
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    38
# maximum <delta-chain-data>/<revision-text-length> ratio
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    39
LIMIT_DELTA2TEXT = 2
1091
d62130f99a73 Move hash function back to revlog from node
mpm@selenic.com
parents: 1089
diff changeset
    40
38637
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    41
class _testrevlog(object):
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    42
    """minimalist fake revlog to use in doctests"""
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    43
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    44
    def __init__(self, data, density=0.5, mingap=0):
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    45
        """data is an list of revision payload boundaries"""
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    46
        self._data = data
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    47
        self._srdensitythreshold = density
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    48
        self._srmingapsize = mingap
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    49
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    50
    def start(self, rev):
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    51
        if rev == 0:
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    52
            return 0
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    53
        return self._data[rev - 1]
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    54
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    55
    def end(self, rev):
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    56
        return self._data[rev]
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    57
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    58
    def length(self, rev):
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    59
        return self.end(rev) - self.start(rev)
e33f784f2a44 revlog: introduce a tiny mock of a revlog class
Boris Feld <boris.feld@octobus.net>
parents: 38636
diff changeset
    60
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
    61
    def __len__(self):
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
    62
        return len(self._data)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
    63
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
    64
def slicechunk(revlog, revs, deltainfo=None, targetsize=None):
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
    65
    """slice revs to reduce the amount of unrelated data to be read from disk.
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
    66
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
    67
    ``revs`` is sliced into groups that should be read in one time.
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
    68
    Assume that revs are sorted.
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    69
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    70
    The initial chunk is sliced until the overall density (payload/chunks-span
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    71
    ratio) is above `revlog._srdensitythreshold`. No gap smaller than
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    72
    `revlog._srmingapsize` is skipped.
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    73
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    74
    If `targetsize` is set, no chunk larger than `targetsize` will be yield.
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    75
    For consistency with other slicing choice, this limit won't go lower than
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    76
    `revlog._srmingapsize`.
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    77
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    78
    If individual revisions chunk are larger than this limit, they will still
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    79
    be raised individually.
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
    80
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    81
    >>> revlog = _testrevlog([
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    82
    ...  5,  #00 (5)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    83
    ...  10, #01 (5)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    84
    ...  12, #02 (2)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    85
    ...  12, #03 (empty)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    86
    ...  27, #04 (15)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    87
    ...  31, #05 (4)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    88
    ...  31, #06 (empty)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    89
    ...  42, #07 (11)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    90
    ...  47, #08 (5)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    91
    ...  47, #09 (empty)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    92
    ...  48, #10 (1)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    93
    ...  51, #11 (3)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    94
    ...  74, #12 (23)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    95
    ...  85, #13 (11)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    96
    ...  86, #14 (1)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    97
    ...  91, #15 (5)
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    98
    ... ])
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
    99
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   100
    >>> list(slicechunk(revlog, list(range(16))))
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
   101
    [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   102
    >>> list(slicechunk(revlog, [0, 15]))
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
   103
    [[0], [15]]
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   104
    >>> list(slicechunk(revlog, [0, 11, 15]))
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
   105
    [[0], [11], [15]]
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   106
    >>> list(slicechunk(revlog, [0, 11, 13, 15]))
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
   107
    [[0], [11, 13, 15]]
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   108
    >>> list(slicechunk(revlog, [1, 2, 3, 5, 8, 10, 11, 14]))
38640
f62b8fb0a484 revlog: document and test _slicechunk
Boris Feld <boris.feld@octobus.net>
parents: 38639
diff changeset
   109
    [[1, 2], [5, 8, 10, 11], [14]]
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   110
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   111
    Slicing with a maximum chunk size
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   112
    >>> list(slicechunk(revlog, [0, 11, 13, 15], targetsize=15))
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   113
    [[0], [11], [13], [15]]
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   114
    >>> list(slicechunk(revlog, [0, 11, 13, 15], targetsize=20))
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   115
    [[0], [11], [13, 15]]
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   116
    """
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   117
    if targetsize is not None:
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   118
        targetsize = max(targetsize, revlog._srmingapsize)
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   119
    # targetsize should not be specified when evaluating delta candidates:
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   120
    # * targetsize is used to ensure we stay within specification when reading,
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   121
    # * deltainfo is used to pick are good delta chain when writing.
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   122
    if not (deltainfo is None or targetsize is None):
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   123
        msg = 'cannot use `targetsize` with a `deltainfo`'
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   124
        raise error.ProgrammingError(msg)
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   125
    for chunk in _slicechunktodensity(revlog, revs,
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   126
                                      deltainfo,
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   127
                                      revlog._srdensitythreshold,
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   128
                                      revlog._srmingapsize):
38643
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   129
        for subchunk in _slicechunktosize(revlog, chunk, targetsize):
967fee55e8d9 revlog: postprocess chunk to slice them down to a certain size
Boris Feld <boris.feld@octobus.net>
parents: 38642
diff changeset
   130
            yield subchunk
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   131
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   132
def _slicechunktosize(revlog, revs, targetsize=None):
38642
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   133
    """slice revs to match the target size
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   134
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   135
    This is intended to be used on chunk that density slicing selected by that
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   136
    are still too large compared to the read garantee of revlog. This might
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   137
    happens when "minimal gap size" interrupted the slicing or when chain are
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   138
    built in a way that create large blocks next to each other.
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   139
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   140
    >>> revlog = _testrevlog([
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   141
    ...  3,  #0 (3)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   142
    ...  5,  #1 (2)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   143
    ...  6,  #2 (1)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   144
    ...  8,  #3 (2)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   145
    ...  8,  #4 (empty)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   146
    ...  11, #5 (3)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   147
    ...  12, #6 (1)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   148
    ...  13, #7 (1)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   149
    ...  14, #8 (1)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   150
    ... ])
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   151
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   152
    Cases where chunk is already small enough
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   153
    >>> list(_slicechunktosize(revlog, [0], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   154
    [[0]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   155
    >>> list(_slicechunktosize(revlog, [6, 7], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   156
    [[6, 7]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   157
    >>> list(_slicechunktosize(revlog, [0], None))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   158
    [[0]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   159
    >>> list(_slicechunktosize(revlog, [6, 7], None))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   160
    [[6, 7]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   161
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   162
    cases where we need actual slicing
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   163
    >>> list(_slicechunktosize(revlog, [0, 1], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   164
    [[0], [1]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   165
    >>> list(_slicechunktosize(revlog, [1, 3], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   166
    [[1], [3]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   167
    >>> list(_slicechunktosize(revlog, [1, 2, 3], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   168
    [[1, 2], [3]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   169
    >>> list(_slicechunktosize(revlog, [3, 5], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   170
    [[3], [5]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   171
    >>> list(_slicechunktosize(revlog, [3, 4, 5], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   172
    [[3], [5]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   173
    >>> list(_slicechunktosize(revlog, [5, 6, 7, 8], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   174
    [[5], [6, 7, 8]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   175
    >>> list(_slicechunktosize(revlog, [0, 1, 2, 3, 4, 5, 6, 7, 8], 3))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   176
    [[0], [1, 2], [3], [5], [6, 7, 8]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   177
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   178
    Case with too large individual chunk (must return valid chunk)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   179
    >>> list(_slicechunktosize(revlog, [0, 1], 2))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   180
    [[0], [1]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   181
    >>> list(_slicechunktosize(revlog, [1, 3], 1))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   182
    [[1], [3]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   183
    >>> list(_slicechunktosize(revlog, [3, 4, 5], 2))
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   184
    [[3], [5]]
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   185
    """
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   186
    assert targetsize is None or 0 <= targetsize
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   187
    if targetsize is None or segmentspan(revlog, revs) <= targetsize:
38642
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   188
        yield revs
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   189
        return
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   190
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   191
    startrevidx = 0
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   192
    startdata = revlog.start(revs[0])
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   193
    endrevidx = 0
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   194
    iterrevs = enumerate(revs)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   195
    next(iterrevs) # skip first rev.
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   196
    for idx, r in iterrevs:
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   197
        span = revlog.end(r) - startdata
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   198
        if span <= targetsize:
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   199
            endrevidx = idx
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   200
        else:
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   201
            chunk = _trimchunk(revlog, revs, startrevidx, endrevidx + 1)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   202
            if chunk:
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   203
                yield chunk
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   204
            startrevidx = idx
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   205
            startdata = revlog.start(r)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   206
            endrevidx = idx
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   207
    yield _trimchunk(revlog, revs, startrevidx)
e59e27e52297 revlog: add function to slice chunk down to a given size
Boris Feld <boris.feld@octobus.net>
parents: 38641
diff changeset
   208
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   209
def _slicechunktodensity(revlog, revs, deltainfo=None, targetdensity=0.5,
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   210
                         mingapsize=0):
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   211
    """slice revs to reduce the amount of unrelated data to be read from disk.
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   212
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   213
    ``revs`` is sliced into groups that should be read in one time.
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   214
    Assume that revs are sorted.
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   215
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   216
    ``deltainfo`` is a _deltainfo instance of a revision that we would append
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   217
    to the top of the revlog.
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   218
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   219
    The initial chunk is sliced until the overall density (payload/chunks-span
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   220
    ratio) is above `targetdensity`. No gap smaller than `mingapsize` is
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   221
    skipped.
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   222
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   223
    >>> revlog = _testrevlog([
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   224
    ...  5,  #00 (5)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   225
    ...  10, #01 (5)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   226
    ...  12, #02 (2)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   227
    ...  12, #03 (empty)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   228
    ...  27, #04 (15)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   229
    ...  31, #05 (4)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   230
    ...  31, #06 (empty)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   231
    ...  42, #07 (11)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   232
    ...  47, #08 (5)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   233
    ...  47, #09 (empty)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   234
    ...  48, #10 (1)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   235
    ...  51, #11 (3)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   236
    ...  74, #12 (23)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   237
    ...  85, #13 (11)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   238
    ...  86, #14 (1)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   239
    ...  91, #15 (5)
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   240
    ... ])
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   241
38655
cd1c484e31e8 revlog: adjust doctest examples to be portable to Python 3
Augie Fackler <augie@google.com>
parents: 38644
diff changeset
   242
    >>> list(_slicechunktodensity(revlog, list(range(16))))
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   243
    [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   244
    >>> list(_slicechunktodensity(revlog, [0, 15]))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   245
    [[0], [15]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   246
    >>> list(_slicechunktodensity(revlog, [0, 11, 15]))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   247
    [[0], [11], [15]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   248
    >>> list(_slicechunktodensity(revlog, [0, 11, 13, 15]))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   249
    [[0], [11, 13, 15]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   250
    >>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14]))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   251
    [[1, 2], [5, 8, 10, 11], [14]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   252
    >>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   253
    ...                           mingapsize=20))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   254
    [[1, 2, 3, 5, 8, 10, 11], [14]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   255
    >>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   256
    ...                           targetdensity=0.95))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   257
    [[1, 2], [5], [8, 10, 11], [14]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   258
    >>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   259
    ...                           targetdensity=0.95, mingapsize=12))
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   260
    [[1, 2], [5, 8, 10, 11], [14]]
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   261
    """
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   262
    start = revlog.start
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   263
    length = revlog.length
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   264
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   265
    if len(revs) <= 1:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   266
        yield revs
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   267
        return
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   268
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   269
    nextrev = len(revlog)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   270
    nextoffset = revlog.end(nextrev - 1)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   271
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   272
    if deltainfo is None:
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   273
        deltachainspan = segmentspan(revlog, revs)
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   274
        chainpayload = sum(length(r) for r in revs)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   275
    else:
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   276
        deltachainspan = deltainfo.distance
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   277
        chainpayload = deltainfo.compresseddeltalen
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   278
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   279
    if deltachainspan < mingapsize:
38635
d083ae26c325 revlog: early return in _slicechunk when span is already small enough
Boris Feld <boris.feld@octobus.net>
parents: 38634
diff changeset
   280
        yield revs
d083ae26c325 revlog: early return in _slicechunk when span is already small enough
Boris Feld <boris.feld@octobus.net>
parents: 38634
diff changeset
   281
        return
d083ae26c325 revlog: early return in _slicechunk when span is already small enough
Boris Feld <boris.feld@octobus.net>
parents: 38634
diff changeset
   282
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   283
    readdata = deltachainspan
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   284
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   285
    if deltachainspan:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   286
        density = chainpayload / float(deltachainspan)
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   287
    else:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   288
        density = 1.0
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   289
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   290
    if density >= targetdensity:
38634
f0ea8b847831 revlog: early return in _slicechunk when density is already good
Paul Morelle <paul.morelle@octobus.net>
parents: 38632
diff changeset
   291
        yield revs
f0ea8b847831 revlog: early return in _slicechunk when density is already good
Paul Morelle <paul.morelle@octobus.net>
parents: 38632
diff changeset
   292
        return
f0ea8b847831 revlog: early return in _slicechunk when density is already good
Paul Morelle <paul.morelle@octobus.net>
parents: 38632
diff changeset
   293
38770
3730b779ed5b sparse-revlog: fix delta validity computation
Boris Feld <boris.feld@octobus.net>
parents: 38763
diff changeset
   294
    if deltainfo is not None and deltainfo.deltalen:
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   295
        revs = list(revs)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   296
        revs.append(nextrev)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   297
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   298
    # Store the gaps in a heap to have them sorted by decreasing size
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   299
    gapsheap = []
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   300
    heapq.heapify(gapsheap)
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   301
    prevend = None
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   302
    for i, rev in enumerate(revs):
38718
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   303
        if rev < nextrev:
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   304
            revstart = start(rev)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   305
            revlen = length(rev)
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   306
        else:
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   307
            revstart = nextoffset
f8762ea73e0d sparse-revlog: implement algorithm to write sparse delta chains (issue5480)
Paul Morelle <paul.morelle@octobus.net>
parents: 38717
diff changeset
   308
            revlen = deltainfo.deltalen
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   309
34898
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   310
        # Skip empty revisions to form larger holes
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   311
        if revlen == 0:
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   312
            continue
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   313
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   314
        if prevend is not None:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   315
            gapsize = revstart - prevend
34881
8c9b08a0c48c sparse-read: skip gaps too small to be worth splitting
Paul Morelle <paul.morelle@octobus.net>
parents: 34880
diff changeset
   316
            # only consider holes that are large enough
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   317
            if gapsize > mingapsize:
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   318
                heapq.heappush(gapsheap, (-gapsize, i))
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   319
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   320
        prevend = revstart + revlen
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   321
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   322
    # Collect the indices of the largest holes until the density is acceptable
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   323
    indicesheap = []
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   324
    heapq.heapify(indicesheap)
38641
feba6be0941b revlog: extract density based slicing into its own function
Boris Feld <boris.feld@octobus.net>
parents: 38640
diff changeset
   325
    while gapsheap and density < targetdensity:
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   326
        oppgapsize, gapidx = heapq.heappop(gapsheap)
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   327
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   328
        heapq.heappush(indicesheap, gapidx)
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   329
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   330
        # the gap sizes are stored as negatives to be sorted decreasingly
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   331
        # by the heap
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   332
        readdata -= (-oppgapsize)
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   333
        if readdata > 0:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   334
            density = chainpayload / float(readdata)
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   335
        else:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   336
            density = 1.0
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   337
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   338
    # Cut the revs at collected indices
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   339
    previdx = 0
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   340
    while indicesheap:
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   341
        idx = heapq.heappop(indicesheap)
34898
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   342
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   343
        chunk = _trimchunk(revlog, revs, previdx, idx)
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   344
        if chunk:
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   345
            yield chunk
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   346
34880
9e18ab7f7240 sparse-read: move from a recursive-based approach to a heap-based one
Boris Feld <boris.feld@octobus.net>
parents: 34825
diff changeset
   347
        previdx = idx
34898
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   348
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   349
    chunk = _trimchunk(revlog, revs, previdx)
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   350
    if chunk:
1bde8e8e5de0 sparse-read: ignore trailing empty revs in each read chunk
Paul Morelle <paul.morelle@octobus.net>
parents: 34881
diff changeset
   351
        yield chunk
34824
e2ad93bcc084 revlog: introduce an experimental flag to slice chunks reads when too sparse
Paul Morelle <paul.morelle@octobus.net>
parents: 34823
diff changeset
   352
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   353
def _trimchunk(revlog, revs, startidx, endidx=None):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   354
    """returns revs[startidx:endidx] without empty trailing revs
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   355
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   356
    Doctest Setup
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   357
    >>> revlog = _testrevlog([
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   358
    ...  5,  #0
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   359
    ...  10, #1
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   360
    ...  12, #2
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   361
    ...  12, #3 (empty)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   362
    ...  17, #4
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   363
    ...  21, #5
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   364
    ...  21, #6 (empty)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   365
    ... ])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   366
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   367
    Contiguous cases:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   368
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   369
    [0, 1, 2, 3, 4, 5]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   370
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0, 5)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   371
    [0, 1, 2, 3, 4]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   372
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0, 4)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   373
    [0, 1, 2]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   374
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 2, 4)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   375
    [2]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   376
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 3)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   377
    [3, 4, 5]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   378
    >>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 3, 5)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   379
    [3, 4]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   380
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   381
    Discontiguous cases:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   382
    >>> _trimchunk(revlog, [1, 3, 5, 6], 0)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   383
    [1, 3, 5]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   384
    >>> _trimchunk(revlog, [1, 3, 5, 6], 0, 2)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   385
    [1]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   386
    >>> _trimchunk(revlog, [1, 3, 5, 6], 1, 3)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   387
    [3, 5]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   388
    >>> _trimchunk(revlog, [1, 3, 5, 6], 1)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   389
    [3, 5]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   390
    """
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   391
    length = revlog.length
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   392
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   393
    if endidx is None:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   394
        endidx = len(revs)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   395
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   396
    # If we have a non-emtpy delta candidate, there are nothing to trim
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   397
    if revs[endidx - 1] < len(revlog):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   398
        # Trim empty revs at the end, except the very first revision of a chain
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   399
        while (endidx > 1
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   400
                and endidx > startidx
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   401
                and length(revs[endidx - 1]) == 0):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   402
            endidx -= 1
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   403
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   404
    return revs[startidx:endidx]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   405
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   406
def segmentspan(revlog, revs, deltainfo=None):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   407
    """Get the byte span of a segment of revisions
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   408
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   409
    revs is a sorted array of revision numbers
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   410
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   411
    >>> revlog = _testrevlog([
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   412
    ...  5,  #0
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   413
    ...  10, #1
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   414
    ...  12, #2
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   415
    ...  12, #3 (empty)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   416
    ...  17, #4
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   417
    ... ])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   418
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   419
    >>> segmentspan(revlog, [0, 1, 2, 3, 4])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   420
    17
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   421
    >>> segmentspan(revlog, [0, 4])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   422
    17
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   423
    >>> segmentspan(revlog, [3, 4])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   424
    5
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   425
    >>> segmentspan(revlog, [1, 2, 3,])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   426
    7
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   427
    >>> segmentspan(revlog, [1, 3])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   428
    7
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   429
    """
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   430
    if not revs:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   431
        return 0
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   432
    if deltainfo is not None and len(revlog) <= revs[-1]:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   433
        if len(revs) == 1:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   434
            return deltainfo.deltalen
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   435
        offset = revlog.end(len(revlog) - 1)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   436
        end = deltainfo.deltalen + offset
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   437
    else:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   438
        end = revlog.end(revs[-1])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   439
    return end - revlog.start(revs[0])
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   440
39358
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   441
def _textfromdelta(fh, revlog, baserev, delta, p1, p2, flags, expectednode):
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   442
    """build full text from a (base, delta) pair and other metadata"""
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   443
    # special case deltas which replace entire base; no need to decode
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   444
    # base revision. this neatly avoids censored bases, which throw when
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   445
    # they're decoded.
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   446
    hlen = struct.calcsize(">lll")
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   447
    if delta[:hlen] == mdiff.replacediffheader(revlog.rawsize(baserev),
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   448
                                               len(delta) - hlen):
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   449
        fulltext = delta[hlen:]
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   450
    else:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   451
        # deltabase is rawtext before changed by flag processors, which is
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   452
        # equivalent to non-raw text
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   453
        basetext = revlog.revision(baserev, _df=fh, raw=False)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   454
        fulltext = mdiff.patch(basetext, delta)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   455
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   456
    try:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   457
        res = revlog._processflags(fulltext, flags, 'read', raw=True)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   458
        fulltext, validatehash = res
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   459
        if validatehash:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   460
            revlog.checkhash(fulltext, expectednode, p1=p1, p2=p2)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   461
        if flags & REVIDX_ISCENSORED:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   462
            raise RevlogError(_('node %s is not censored') % expectednode)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   463
    except CensoredNodeError:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   464
        # must pass the censored index flag to add censored revisions
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   465
        if not flags & REVIDX_ISCENSORED:
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   466
            raise
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   467
    return fulltext
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   468
35638
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   469
@attr.s(slots=True, frozen=True)
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   470
class _deltainfo(object):
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   471
    distance = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   472
    deltalen = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   473
    data = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   474
    base = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   475
    chainbase = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   476
    chainlen = attr.ib()
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   477
    compresseddeltalen = attr.ib()
39187
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   478
    snapshotdepth = attr.ib()
35638
edc9330acac1 revlog: introduce 'deltainfo' to distinguish from 'delta'
Paul Morelle <paul.morelle@octobus.net>
parents: 35637
diff changeset
   479
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   480
def isgooddeltainfo(revlog, deltainfo, revinfo):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   481
    """Returns True if the given delta is good. Good means that it is within
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   482
    the disk span, disk size, and chain length bounds that we know to be
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   483
    performant."""
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   484
    if deltainfo is None:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   485
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   486
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   487
    # - 'deltainfo.distance' is the distance from the base revision --
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   488
    #   bounding it limits the amount of I/O we need to do.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   489
    # - 'deltainfo.compresseddeltalen' is the sum of the total size of
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   490
    #   deltas we need to apply -- bounding it limits the amount of CPU
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   491
    #   we consume.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   492
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   493
    if revlog._sparserevlog:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   494
        # As sparse-read will be used, we can consider that the distance,
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   495
        # instead of being the span of the whole chunk,
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   496
        # is the span of the largest read chunk
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   497
        base = deltainfo.base
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   498
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   499
        if base != nullrev:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   500
            deltachain = revlog._deltachain(base)[0]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   501
        else:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   502
            deltachain = []
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   503
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   504
        # search for the first non-snapshot revision
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   505
        for idx, r in enumerate(deltachain):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   506
            if not revlog.issnapshot(r):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   507
                break
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   508
        deltachain = deltachain[idx:]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   509
        chunks = slicechunk(revlog, deltachain, deltainfo)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   510
        all_span = [segmentspan(revlog, revs, deltainfo)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   511
                    for revs in chunks]
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   512
        distance = max(all_span)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   513
    else:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   514
        distance = deltainfo.distance
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   515
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   516
    textlen = revinfo.textlen
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   517
    defaultmax = textlen * 4
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   518
    maxdist = revlog._maxdeltachainspan
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   519
    if not maxdist:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   520
        maxdist = distance # ensure the conditional pass
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   521
    maxdist = max(maxdist, defaultmax)
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   522
    if revlog._sparserevlog and maxdist < revlog._srmingapsize:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   523
        # In multiple place, we are ignoring irrelevant data range below a
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   524
        # certain size. Be also apply this tradeoff here and relax span
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   525
        # constraint for small enought content.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   526
        maxdist = revlog._srmingapsize
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   527
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   528
    # Bad delta from read span:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   529
    #
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   530
    #   If the span of data read is larger than the maximum allowed.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   531
    if maxdist < distance:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   532
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   533
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   534
    # Bad delta from new delta size:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   535
    #
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   536
    #   If the delta size is larger than the target text, storing the
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   537
    #   delta will be inefficient.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   538
    if textlen < deltainfo.deltalen:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   539
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   540
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   541
    # Bad delta from cumulated payload size:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   542
    #
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   543
    #   If the sum of delta get larger than K * target text length.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   544
    if textlen * LIMIT_DELTA2TEXT < deltainfo.compresseddeltalen:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   545
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   546
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   547
    # Bad delta from chain length:
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   548
    #
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   549
    #   If the number of delta in the chain gets too high.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   550
    if (revlog._maxchainlen
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   551
            and revlog._maxchainlen < deltainfo.chainlen):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   552
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   553
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   554
    # bad delta from intermediate snapshot size limit
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   555
    #
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   556
    #   If an intermediate snapshot size is higher than the limit.  The
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   557
    #   limit exist to prevent endless chain of intermediate delta to be
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   558
    #   created.
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   559
    if (deltainfo.snapshotdepth is not None and
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   560
            (textlen >> deltainfo.snapshotdepth) < deltainfo.deltalen):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   561
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   562
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   563
    # bad delta if new intermediate snapshot is larger than the previous
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   564
    # snapshot
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   565
    if (deltainfo.snapshotdepth
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   566
            and revlog.length(deltainfo.base) < deltainfo.deltalen):
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   567
        return False
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   568
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   569
    return True
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   570
39361
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   571
def _candidategroups(revlog, p1, p2, cachedelta):
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   572
    """
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   573
    Provides revisions that present an interest to be diffed against,
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   574
    grouped by level of easiness.
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   575
    """
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   576
    gdelta = revlog._generaldelta
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   577
    curr = len(revlog)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   578
    prev = curr - 1
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   579
    p1r, p2r = revlog.rev(p1), revlog.rev(p2)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   580
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   581
    # should we try to build a delta?
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   582
    if prev != nullrev and revlog._storedeltachains:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   583
        tested = set()
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   584
        # This condition is true most of the time when processing
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   585
        # changegroup data into a generaldelta repo. The only time it
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   586
        # isn't true is if this is the first revision in a delta chain
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   587
        # or if ``format.generaldelta=true`` disabled ``lazydeltabase``.
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   588
        if cachedelta and gdelta and revlog._lazydeltabase:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   589
            # Assume what we received from the server is a good choice
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   590
            # build delta will reuse the cache
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   591
            yield (cachedelta[0],)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   592
            tested.add(cachedelta[0])
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   593
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   594
        if gdelta:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   595
            # exclude already lazy tested base if any
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   596
            parents = [p for p in (p1r, p2r)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   597
                       if p != nullrev and p not in tested]
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   598
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   599
            if not revlog._deltabothparents and len(parents) == 2:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   600
                parents.sort()
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   601
                # To minimize the chance of having to build a fulltext,
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   602
                # pick first whichever parent is closest to us (max rev)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   603
                yield (parents[1],)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   604
                # then the other one (min rev) if the first did not fit
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   605
                yield (parents[0],)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   606
                tested.update(parents)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   607
            elif len(parents) > 0:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   608
                # Test all parents (1 or 2), and keep the best candidate
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   609
                yield parents
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   610
                tested.update(parents)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   611
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   612
        if prev not in tested:
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   613
            # other approach failed try against prev to hopefully save us a
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   614
            # fulltext.
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   615
            yield (prev,)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   616
            tested.add(prev)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   617
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   618
class deltacomputer(object):
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   619
    def __init__(self, revlog):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   620
        self.revlog = revlog
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   621
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   622
    def buildtext(self, revinfo, fh):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   623
        """Builds a fulltext version of a revision
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   624
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   625
        revinfo: _revisioninfo instance that contains all needed info
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   626
        fh:      file handle to either the .i or the .d revlog file,
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   627
                 depending on whether it is inlined or not
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   628
        """
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   629
        btext = revinfo.btext
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   630
        if btext[0] is not None:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   631
            return btext[0]
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   632
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   633
        revlog = self.revlog
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   634
        cachedelta = revinfo.cachedelta
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   635
        baserev = cachedelta[0]
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   636
        delta = cachedelta[1]
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   637
39358
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   638
        fulltext = btext[0] = _textfromdelta(fh, revlog, baserev, delta,
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   639
                                             revinfo.p1, revinfo.p2,
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   640
                                             revinfo.flags, revinfo.node)
fd0150a3c2fe revlogdeltas: extra fulltext building in its own function
Boris Feld <boris.feld@octobus.net>
parents: 39357
diff changeset
   641
        return fulltext
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   642
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   643
    def _builddeltadiff(self, base, revinfo, fh):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   644
        revlog = self.revlog
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   645
        t = self.buildtext(revinfo, fh)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   646
        if revlog.iscensored(base):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   647
            # deltas based on a censored revision must replace the
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   648
            # full content in one patch, so delta works everywhere
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   649
            header = mdiff.replacediffheader(revlog.rawsize(base), len(t))
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   650
            delta = header + t
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   651
        else:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   652
            ptext = revlog.revision(base, _df=fh, raw=True)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   653
            delta = mdiff.textdiff(ptext, t)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   654
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   655
        return delta
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   656
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   657
    def _builddeltainfo(self, revinfo, base, fh):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   658
        # can we use the cached delta?
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   659
        if revinfo.cachedelta and revinfo.cachedelta[0] == base:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   660
            delta = revinfo.cachedelta[1]
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   661
        else:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   662
            delta = self._builddeltadiff(base, revinfo, fh)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   663
        revlog = self.revlog
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   664
        header, data = revlog.compress(delta)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   665
        deltalen = len(header) + len(data)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   666
        chainbase = revlog.chainbase(base)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   667
        offset = revlog.end(len(revlog) - 1)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   668
        dist = deltalen + offset - revlog.start(chainbase)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   669
        if revlog._generaldelta:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   670
            deltabase = base
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   671
        else:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   672
            deltabase = chainbase
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   673
        chainlen, compresseddeltalen = revlog._chaininfo(base)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   674
        chainlen += 1
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   675
        compresseddeltalen += deltalen
39187
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   676
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   677
        revlog = self.revlog
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   678
        snapshotdepth = None
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   679
        if deltabase == nullrev:
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   680
            snapshotdepth = 0
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   681
        elif revlog._sparserevlog and revlog.issnapshot(deltabase):
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   682
            # A delta chain should always be one full snapshot,
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   683
            # zero or more semi-snapshots, and zero or more deltas
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   684
            p1, p2 = revlog.rev(revinfo.p1), revlog.rev(revinfo.p2)
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   685
            if deltabase not in (p1, p2) and revlog.issnapshot(deltabase):
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   686
                snapshotdepth = len(revlog._deltachain(deltabase)[0])
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   687
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   688
        return _deltainfo(dist, deltalen, (header, data), deltabase,
39187
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   689
                          chainbase, chainlen, compresseddeltalen,
e0da43e2f71f revlog: compute snapshot depth on delta info
Boris Feld <boris.feld@octobus.net>
parents: 39185
diff changeset
   690
                          snapshotdepth)
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   691
39360
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   692
    def _fullsnapshotinfo(self, fh, revinfo):
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   693
        curr = len(self.revlog)
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   694
        rawtext = self.buildtext(revinfo, fh)
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   695
        data = self.revlog.compress(rawtext)
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   696
        compresseddeltalen = deltalen = dist = len(data[1]) + len(data[0])
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   697
        deltabase = chainbase = curr
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   698
        snapshotdepth = 0
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   699
        chainlen = 1
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   700
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   701
        return _deltainfo(dist, deltalen, data, deltabase,
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   702
                          chainbase, chainlen, compresseddeltalen,
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   703
                          snapshotdepth)
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   704
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   705
    def finddeltainfo(self, revinfo, fh):
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   706
        """Find an acceptable delta against a candidate revision
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   707
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   708
        revinfo: information about the revision (instance of _revisioninfo)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   709
        fh:      file handle to either the .i or the .d revlog file,
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   710
                 depending on whether it is inlined or not
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   711
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   712
        Returns the first acceptable candidate revision, as ordered by
39361
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   713
        _candidategroups
39360
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   714
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   715
        If no suitable deltabase is found, we return delta info for a full
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   716
        snapshot.
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   717
        """
39122
dbb3e9e44fce revlog: do not search for delta for empty content
Boris Feld <boris.feld@octobus.net>
parents: 39121
diff changeset
   718
        if not revinfo.textlen:
39360
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   719
            return self._fullsnapshotinfo(fh, revinfo)
39122
dbb3e9e44fce revlog: do not search for delta for empty content
Boris Feld <boris.feld@octobus.net>
parents: 39121
diff changeset
   720
39359
6f4b8f607a31 revlogdeltas: move special cases around raw revisions in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39358
diff changeset
   721
        # no delta for flag processor revision (see "candelta" for why)
6f4b8f607a31 revlogdeltas: move special cases around raw revisions in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39358
diff changeset
   722
        # not calling candelta since only one revision needs test, also to
6f4b8f607a31 revlogdeltas: move special cases around raw revisions in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39358
diff changeset
   723
        # avoid overhead fetching flags again.
6f4b8f607a31 revlogdeltas: move special cases around raw revisions in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39358
diff changeset
   724
        if revinfo.flags & REVIDX_RAWTEXT_CHANGING_FLAGS:
39360
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   725
            return self._fullsnapshotinfo(fh, revinfo)
39359
6f4b8f607a31 revlogdeltas: move special cases around raw revisions in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39358
diff changeset
   726
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   727
        cachedelta = revinfo.cachedelta
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   728
        p1 = revinfo.p1
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   729
        p2 = revinfo.p2
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   730
        revlog = self.revlog
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   731
39120
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   732
        deltalength = self.revlog.length
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   733
        deltaparent = self.revlog.deltaparent
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   734
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   735
        deltainfo = None
39124
f90b333e79cb revlog: filter out "invalid" delta base candidates
Boris Feld <boris.feld@octobus.net>
parents: 39123
diff changeset
   736
        deltas_limit = revinfo.textlen * LIMIT_DELTA2TEXT
39361
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   737
        groups = _candidategroups(self.revlog, p1, p2, cachedelta)
507f5b1dd7c8 revlogdeltas: extract _getcandidaterevs in a function
Boris Feld <boris.feld@octobus.net>
parents: 39360
diff changeset
   738
        for candidaterevs in groups:
39124
f90b333e79cb revlog: filter out "invalid" delta base candidates
Boris Feld <boris.feld@octobus.net>
parents: 39123
diff changeset
   739
            # filter out delta base that will never produce good delta
f90b333e79cb revlog: filter out "invalid" delta base candidates
Boris Feld <boris.feld@octobus.net>
parents: 39123
diff changeset
   740
            candidaterevs = [r for r in candidaterevs
f90b333e79cb revlog: filter out "invalid" delta base candidates
Boris Feld <boris.feld@octobus.net>
parents: 39123
diff changeset
   741
                             if self.revlog.length(r) <= deltas_limit]
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   742
            nominateddeltas = []
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   743
            for candidaterev in candidaterevs:
39120
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   744
                # skip over empty delta (no need to include them in a chain)
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   745
                while candidaterev != nullrev and not deltalength(candidaterev):
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   746
                    candidaterev = deltaparent(candidaterev)
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   747
                # no need to try a delta against nullid, this will be handled
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   748
                # by fulltext later.
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   749
                if candidaterev == nullrev:
8f83a953dddf revlog: skip over empty revision when looking for delta base
Boris Feld <boris.feld@octobus.net>
parents: 39116
diff changeset
   750
                    continue
35840
33275ab5e837 revlog: do not use delta for lfs revisions
Jun Wu <quark@fb.com>
parents: 35839
diff changeset
   751
                # no delta for rawtext-changing revs (see "candelta" for why)
33275ab5e837 revlog: do not use delta for lfs revisions
Jun Wu <quark@fb.com>
parents: 35839
diff changeset
   752
                if revlog.flags(candidaterev) & REVIDX_RAWTEXT_CHANGING_FLAGS:
33275ab5e837 revlog: do not use delta for lfs revisions
Jun Wu <quark@fb.com>
parents: 35839
diff changeset
   753
                    continue
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   754
                candidatedelta = self._builddeltainfo(revinfo, candidaterev, fh)
39357
655b5b465953 revlog: split functionality related to deltas computation in a new module
Boris Feld <boris.feld@octobus.net>
parents: 39356
diff changeset
   755
                if isgooddeltainfo(self.revlog, candidatedelta, revinfo):
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   756
                    nominateddeltas.append(candidatedelta)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   757
            if nominateddeltas:
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   758
                deltainfo = min(nominateddeltas, key=lambda x: x.deltalen)
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   759
                break
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   760
39360
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   761
        if deltainfo is None:
5d343a24bff5 revlogdeltas: always return a delta info object in finddeltainfo
Boris Feld <boris.feld@octobus.net>
parents: 39359
diff changeset
   762
            deltainfo = self._fullsnapshotinfo(fh, revinfo)
35738
f90f6fd130c1 revlog: group delta computation methods under _deltacomputer object
Paul Morelle <paul.morelle@octobus.net>
parents: 35737
diff changeset
   763
        return deltainfo