mercurial/changegroup.py
author Gregory Szorc <gregory.szorc@gmail.com>
Wed, 08 Aug 2018 14:19:02 -0700
changeset 39009 9e8eb2b444e5
parent 39008 8c84f1ef949e
child 39010 fcdab6629dde
permissions -rw-r--r--
changegroup: extract cgpacker.group() to standalone function It doesn't need to be part of the packer class. Differential Revision: https://phab.mercurial-scm.org/D4206
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
8226
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     1
# changegroup.py - Mercurial changegroup manipulation functions
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     2
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     3
#  Copyright 2006 Matt Mackall <mpm@selenic.com>
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     4
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     5
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9437
diff changeset
     6
# GNU General Public License version 2 or any later version.
3877
abaee83ce0a6 Replace demandload with new demandimport
Matt Mackall <mpm@selenic.com>
parents: 3859
diff changeset
     7
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
     8
from __future__ import absolute_import
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
     9
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    10
import os
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    11
import struct
20933
d3775db748a0 localrepo: move the addchangegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20932
diff changeset
    12
import weakref
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    13
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    14
from .i18n import _
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    15
from .node import (
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    16
    hex,
38883
ee1ea96cf9c9 changegroup: move ellipsisdata() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38835
diff changeset
    17
    nullid,
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    18
    nullrev,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    19
    short,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    20
)
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    21
38893
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
    22
from .thirdparty import (
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
    23
    attr,
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
    24
)
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
    25
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    26
from . import (
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    27
    dagutil,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    28
    error,
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
    29
    match as matchmod,
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    30
    mdiff,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    31
    phases,
30925
82f1ef8b4477 py3: convert the mode argument of os.fdopen to unicodes (2 of 2)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30743
diff changeset
    32
    pycompat,
38835
a232e6744ba3 narrow: move requirement constant from changegroup to repository
Martin von Zweigbergk <martinvonz@google.com>
parents: 38806
diff changeset
    33
    repository,
38883
ee1ea96cf9c9 changegroup: move ellipsisdata() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38835
diff changeset
    34
    revlog,
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    35
    util,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    36
)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    37
37084
f0b6fbea00cf stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents: 36835
diff changeset
    38
from .utils import (
f0b6fbea00cf stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents: 36835
diff changeset
    39
    stringutil,
f0b6fbea00cf stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents: 36835
diff changeset
    40
)
f0b6fbea00cf stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents: 36835
diff changeset
    41
38896
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
    42
_CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
    43
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
    44
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
    45
37132
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
    46
LFS_REQUIREMENT = 'lfs'
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
    47
35754
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35012
diff changeset
    48
readexactly = util.readexactly
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    49
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    50
def getchunk(stream):
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    51
    """return the next chunk from stream as a string"""
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    52
    d = readexactly(stream, 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    53
    l = struct.unpack(">l", d)[0]
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    54
    if l <= 4:
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
    55
        if l:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
    56
            raise error.Abort(_("invalid chunk length %d") % l)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    57
        return ""
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    58
    return readexactly(stream, l - 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    59
5368
61462e7d62ed changegroup: avoid large copies
Matt Mackall <mpm@selenic.com>
parents: 3932
diff changeset
    60
def chunkheader(length):
9437
1c4e4004f3a6 Improve some docstrings relating to changegroups and prepush().
Greg Ward <greg-hg@gerg.ca>
parents: 9087
diff changeset
    61
    """return a changegroup chunk header (string)"""
5368
61462e7d62ed changegroup: avoid large copies
Matt Mackall <mpm@selenic.com>
parents: 3932
diff changeset
    62
    return struct.pack(">l", length + 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    63
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    64
def closechunk():
9437
1c4e4004f3a6 Improve some docstrings relating to changegroups and prepush().
Greg Ward <greg-hg@gerg.ca>
parents: 9087
diff changeset
    65
    """return a changegroup chunk header (string) for a zero-length chunk"""
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    66
    return struct.pack(">l", 0)
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    67
38981
227ebd88ce5e changegroup: pull _fileheader out of cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38980
diff changeset
    68
def _fileheader(path):
227ebd88ce5e changegroup: pull _fileheader out of cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38980
diff changeset
    69
    """Obtain a changegroup chunk header for a named path."""
227ebd88ce5e changegroup: pull _fileheader out of cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38980
diff changeset
    70
    return chunkheader(len(path)) + path
227ebd88ce5e changegroup: pull _fileheader out of cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38980
diff changeset
    71
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    72
def writechunks(ui, chunks, filename, vfs=None):
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    73
    """Write chunks to a file and return its filename.
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    74
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    75
    The stream is assumed to be a bundle file.
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    76
    Existing files will not be overwritten.
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    77
    If no filename is specified, a temporary file is created.
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    78
    """
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    79
    fh = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    80
    cleanup = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    81
    try:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    82
        if filename:
20976
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
    83
            if vfs:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
    84
                fh = vfs.open(filename, "wb")
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
    85
            else:
30212
260af19891f2 changegroup: increase write buffer size to 128k
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30211
diff changeset
    86
                # Increase default buffer size because default is usually
260af19891f2 changegroup: increase write buffer size to 128k
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30211
diff changeset
    87
                # small (4k is common on Linux).
260af19891f2 changegroup: increase write buffer size to 128k
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30211
diff changeset
    88
                fh = open(filename, "wb", 131072)
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    89
        else:
38164
aac4be30e250 py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents: 37339
diff changeset
    90
            fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
36835
5bc7ff103081 py3: use r'' instead of sysstr('') to get around code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 36760
diff changeset
    91
            fh = os.fdopen(fd, r"wb")
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    92
        cleanup = filename
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    93
        for c in chunks:
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    94
            fh.write(c)
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    95
        cleanup = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    96
        return filename
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    97
    finally:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    98
        if fh is not None:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    99
            fh.close()
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   100
        if cleanup is not None:
20976
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   101
            if filename and vfs:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   102
                vfs.unlink(cleanup)
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   103
            else:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   104
                os.unlink(cleanup)
3660
8500a13ec44b create a readbundle function
Matt Mackall <mpm@selenic.com>
parents: 3659
diff changeset
   105
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   106
class cg1unpacker(object):
26708
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   107
    """Unpacker for cg1 changegroup streams.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   108
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   109
    A changegroup unpacker handles the framing of the revision data in
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   110
    the wire format. Most consumers will want to use the apply()
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   111
    method to add the changes from the changegroup to a repository.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   112
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   113
    If you're forwarding a changegroup unmodified to another consumer,
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   114
    use getchunks(), which returns an iterator of changegroup
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   115
    chunks. This is mostly useful for cases where you need to know the
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   116
    data stream has ended by observing the end of the changegroup.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   117
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   118
    deltachunk() is useful only if you're applying delta data. Most
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   119
    consumers should prefer apply() instead.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   120
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   121
    A few other public methods exist. Those are used only for
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   122
    bundlerepo and some debug commands - their use is discouraged.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   123
    """
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   124
    deltaheader = _CHANGEGROUPV1_DELTA_HEADER
38896
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
   125
    deltaheadersize = deltaheader.size
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   126
    version = '01'
27920
da5f23362517 changegroup: cg3 has two empty groups *after* manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27867
diff changeset
   127
    _grouplistcount = 1 # One list of files after the manifests
da5f23362517 changegroup: cg3 has two empty groups *after* manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27867
diff changeset
   128
29593
953839de96ab bundle2: store changeset count when creating file bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29371
diff changeset
   129
    def __init__(self, fh, alg, extras=None):
30354
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   130
        if alg is None:
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   131
            alg = 'UN'
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   132
        if alg not in util.compengines.supportedbundletypes:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   133
            raise error.Abort(_('unknown stream compression type: %s')
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
   134
                             % alg)
26392
127b59787fd5 changegroup: use a different compression key for BZ in HG10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26272
diff changeset
   135
        if alg == 'BZ':
127b59787fd5 changegroup: use a different compression key for BZ in HG10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26272
diff changeset
   136
            alg = '_truncatedBZ'
30354
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   137
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   138
        compengine = util.compengines.forbundletype(alg)
a37a96d838b9 changegroup: use compression engines API
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30339
diff changeset
   139
        self._stream = compengine.decompressorreader(fh)
12044
bcc7139521b7 bundlerepo: remove duplication of bundle decompressors
Matt Mackall <mpm@selenic.com>
parents: 12043
diff changeset
   140
        self._type = alg
29593
953839de96ab bundle2: store changeset count when creating file bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29371
diff changeset
   141
        self.extras = extras or {}
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   142
        self.callback = None
26706
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   143
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   144
    # These methods (compressed, read, seek, tell) all appear to only
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   145
    # be used by bundlerepo, but it's a little hard to tell.
12044
bcc7139521b7 bundlerepo: remove duplication of bundle decompressors
Matt Mackall <mpm@selenic.com>
parents: 12043
diff changeset
   146
    def compressed(self):
30589
182cacaa4c32 cg1packer: fix `compressed` method
Stanislau Hlebik <stash@fb.com>
parents: 30354
diff changeset
   147
        return self._type is not None and self._type != 'UN'
12043
bef5effb3db0 bundle: introduce bundle class
Matt Mackall <mpm@selenic.com>
parents: 12042
diff changeset
   148
    def read(self, l):
bef5effb3db0 bundle: introduce bundle class
Matt Mackall <mpm@selenic.com>
parents: 12042
diff changeset
   149
        return self._stream.read(l)
12330
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   150
    def seek(self, pos):
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   151
        return self._stream.seek(pos)
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   152
    def tell(self):
12332
680fe77ab5b8 bundlerepo: use bundle objects everywhere
Matt Mackall <mpm@selenic.com>
parents: 12330
diff changeset
   153
        return self._stream.tell()
12347
6277a9469dff bundlerepo: restore close() method
Matt Mackall <mpm@selenic.com>
parents: 12336
diff changeset
   154
    def close(self):
6277a9469dff bundlerepo: restore close() method
Matt Mackall <mpm@selenic.com>
parents: 12336
diff changeset
   155
        return self._stream.close()
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   156
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   157
    def _chunklength(self):
13459
acbe171c8fbe changegroup: fix typo introduced in 9f2c407caf34
Jim Hague <jim.hague@acm.org>
parents: 13458
diff changeset
   158
        d = readexactly(self._stream, 4)
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   159
        l = struct.unpack(">l", d)[0]
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   160
        if l <= 4:
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   161
            if l:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   162
                raise error.Abort(_("invalid chunk length %d") % l)
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   163
            return 0
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   164
        if self.callback:
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   165
            self.callback()
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   166
        return l - 4
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   167
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   168
    def changelogheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   169
        """v10 does not have a changelog header chunk"""
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   170
        return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   171
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   172
    def manifestheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   173
        """v10 does not have a manifest header chunk"""
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   174
        return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   175
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   176
    def filelogheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   177
        """return the header of the filelogs chunk, v10 only has the filename"""
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   178
        l = self._chunklength()
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   179
        if not l:
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   180
            return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   181
        fname = readexactly(self._stream, l)
20675
f8d50add83e1 changegroup: move from dict() construction to {} literals
Augie Fackler <raf@durin42.com>
parents: 19708
diff changeset
   182
        return {'filename': fname}
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   183
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   184
    def _deltaheader(self, headertuple, prevnode):
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   185
        node, p1, p2, cs = headertuple
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   186
        if prevnode is None:
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   187
            deltabase = p1
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   188
        else:
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   189
            deltabase = prevnode
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   190
        flags = 0
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   191
        return node, p1, p2, deltabase, cs, flags
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   192
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   193
    def deltachunk(self, prevnode):
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   194
        l = self._chunklength()
12336
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   195
        if not l:
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   196
            return {}
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   197
        headerdata = readexactly(self._stream, self.deltaheadersize)
38896
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
   198
        header = self.deltaheader.unpack(headerdata)
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   199
        delta = readexactly(self._stream, l - self.deltaheadersize)
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   200
        node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
34294
05131c963767 changegroup: remove dictionary creation from deltachunk
Durham Goode <durham@fb.com>
parents: 34291
diff changeset
   201
        return (node, p1, p2, cs, deltabase, delta, flags)
12336
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   202
20999
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   203
    def getchunks(self):
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   204
        """returns all the chunks contains in the bundle
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   205
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   206
        Used when you need to forward the binary stream to a file or another
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   207
        network API. To do so, it parse the changegroup data, otherwise it will
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   208
        block in case of sshrepo because it don't know the end of the stream.
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   209
        """
34091
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   210
        # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   211
        # and a list of filelogs. For changegroup 3, we expect 4 parts:
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   212
        # changelog, manifestlog, a list of tree manifestlogs, and a list of
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   213
        # filelogs.
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   214
        #
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   215
        # Changelog and manifestlog parts are terminated with empty chunks. The
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   216
        # tree and file parts are a list of entry sections. Each entry section
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   217
        # is a series of chunks terminating in an empty chunk. The list of these
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   218
        # entry sections is terminated in yet another empty chunk, so we know
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   219
        # we've reached the end of the tree/file list when we reach an empty
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   220
        # chunk that was proceeded by no non-empty chunks.
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   221
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   222
        parts = 0
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   223
        while parts < 2 + self._grouplistcount:
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   224
            noentries = True
20999
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   225
            while True:
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   226
                chunk = getchunk(self)
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   227
                if not chunk:
34091
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   228
                    # The first two empty chunks represent the end of the
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   229
                    # changelog and the manifestlog portions. The remaining
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   230
                    # empty chunks represent either A) the end of individual
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   231
                    # tree or file entries in the file list, or B) the end of
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   232
                    # the entire list. It's the end of the entire list if there
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   233
                    # were no entries (i.e. noentries is True).
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   234
                    if parts < 2:
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   235
                        parts += 1
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   236
                    elif noentries:
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   237
                        parts += 1
20999
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   238
                    break
34091
bbdca7e460c0 changegroup: fix to allow empty manifest parts
Durham Goode <durham@fb.com>
parents: 33712
diff changeset
   239
                noentries = False
20999
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   240
                yield chunkheader(len(chunk))
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   241
                pos = 0
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   242
                while pos < len(chunk):
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   243
                    next = pos + 2**20
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   244
                    yield chunk[pos:next]
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   245
                    pos = next
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   246
            yield closechunk()
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   247
38346
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   248
    def _unpackmanifests(self, repo, revmap, trp, prog):
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   249
        self.callback = prog.increment
26712
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   250
        # no need to check for empty manifest group here:
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   251
        # if the result of the merge of 1 and 2 is the same in 3 and 4,
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   252
        # no new manifest will be created and the manifest group will
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   253
        # be empty during the pull
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   254
        self.manifestheader()
34291
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   255
        deltas = self.deltaiter()
38556
0db41eb0a3ac manifest: define and implement addgroup() on manifestlog
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38410
diff changeset
   256
        repo.manifestlog.addgroup(deltas, revmap, trp)
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38346
diff changeset
   257
        prog.complete()
28360
11287888ce4b changegroup: exclude submanifests from manifest progress
Martin von Zweigbergk <martinvonz@google.com>
parents: 28281
diff changeset
   258
        self.callback = None
26712
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   259
33308
248d5890c80a changegroup: remove option to allow empty changegroup (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33227
diff changeset
   260
    def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
248d5890c80a changegroup: remove option to allow empty changegroup (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33227
diff changeset
   261
              expectedtotal=None):
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   262
        """Add the changegroup returned by source.read() to this repo.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   263
        srctype is a string like 'push', 'pull', or 'unbundle'.  url is
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   264
        the URL of the repo where this changegroup is coming from.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   265
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   266
        Return an integer summarizing the change to this repo:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   267
        - nothing changed or no source: 0
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   268
        - more heads than before: 1+added heads (2..n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   269
        - fewer heads than before: -1-removed heads (-2..-n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   270
        - number of heads stays the same: 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   271
        """
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   272
        repo = repo.unfiltered()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   273
        def csmap(x):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   274
            repo.ui.debug("add changeset %s\n" % short(x))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   275
            return len(cl)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   276
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   277
        def revmap(x):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   278
            return cl.rev(x)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   279
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   280
        changesets = files = revisions = 0
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   281
26880
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   282
        try:
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   283
            # The transaction may already carry source information. In this
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   284
            # case we use the top level data. We overwrite the argument
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   285
            # because we need to use the top level value (if they exist)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   286
            # in this function.
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   287
            srctype = tr.hookargs.setdefault('source', srctype)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   288
            url = tr.hookargs.setdefault('url', url)
33671
38fc45721334 changegroup: wrap some ** expansions in strkwargs
Augie Fackler <augie@google.com>
parents: 33461
diff changeset
   289
            repo.hook('prechangegroup',
38fc45721334 changegroup: wrap some ** expansions in strkwargs
Augie Fackler <augie@google.com>
parents: 33461
diff changeset
   290
                      throw=True, **pycompat.strkwargs(tr.hookargs))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   291
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   292
            # write changelog data to temp files so concurrent readers
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   293
            # will not see an inconsistent view
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   294
            cl = repo.changelog
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   295
            cl.delayupdate(tr)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   296
            oldheads = set(cl.heads())
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   297
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   298
            trp = weakref.proxy(tr)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   299
            # pull off the changeset group
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   300
            repo.ui.status(_("adding changesets\n"))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   301
            clstart = len(cl)
38346
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   302
            progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   303
                                            total=expectedtotal)
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   304
            self.callback = progress.increment
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   305
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   306
            efiles = set()
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   307
            def onchangelog(cl, node):
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   308
                efiles.update(cl.readfiles(node))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   309
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   310
            self.changelogheader()
34291
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   311
            deltas = self.deltaiter()
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   312
            cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   313
            efiles = len(efiles)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   314
33308
248d5890c80a changegroup: remove option to allow empty changegroup (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33227
diff changeset
   315
            if not cgnodes:
33309
69c4493a54f9 changegroup: don't fail on empty changegroup (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33308
diff changeset
   316
                repo.ui.develwarn('applied empty changegroup',
34734
3572b2031cec devel-warn: add 'warn-' to 'devel.empty-changegroup' config
Boris Feld <boris.feld@octobus.net>
parents: 34294
diff changeset
   317
                                  config='warn-empty-changegroup')
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   318
            clend = len(cl)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   319
            changesets = clend - clstart
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38346
diff changeset
   320
            progress.complete()
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   321
            self.callback = None
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   322
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   323
            # pull off the manifest group
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   324
            repo.ui.status(_("adding manifests\n"))
38346
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   325
            # We know that we'll never have more manifests than we had
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   326
            # changesets.
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   327
            progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   328
                                            total=changesets)
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   329
            self._unpackmanifests(repo, revmap, trp, progress)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   330
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   331
            needfiles = {}
33227
86c9aa1d598f configitems: register the 'server.validate' config
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33183
diff changeset
   332
            if repo.ui.configbool('server', 'validate'):
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   333
                cl = repo.changelog
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   334
                ml = repo.manifestlog
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   335
                # validate incoming csets have their manifests
38783
e7aa113b14f7 global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38556
diff changeset
   336
                for cset in pycompat.xrange(clstart, clend):
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   337
                    mfnode = cl.changelogrevision(cset).manifest
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   338
                    mfest = ml[mfnode].readdelta()
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   339
                    # store file cgnodes we must see
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   340
                    for f, n in mfest.iteritems():
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   341
                        needfiles.setdefault(f, set()).add(n)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   342
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   343
            # process the files
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   344
            repo.ui.status(_("adding file changes\n"))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   345
            newrevs, newfiles = _addchangegroupfiles(
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   346
                repo, self, revmap, trp, efiles, needfiles)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   347
            revisions += newrevs
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   348
            files += newfiles
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   349
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   350
            deltaheads = 0
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   351
            if oldheads:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   352
                heads = cl.heads()
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   353
                deltaheads = len(heads) - len(oldheads)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   354
                for h in heads:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   355
                    if h not in oldheads and repo[h].closesbranch():
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   356
                        deltaheads -= 1
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   357
            htext = ""
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   358
            if deltaheads:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   359
                htext = _(" (%+d heads)") % deltaheads
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   360
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   361
            repo.ui.status(_("added %d changesets"
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   362
                             " with %d changes to %d files%s\n")
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   363
                             % (changesets, revisions, files, htext))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   364
            repo.invalidatevolatilesets()
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   365
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   366
            if changesets > 0:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   367
                if 'node' not in tr.hookargs:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   368
                    tr.hookargs['node'] = hex(cl.node(clstart))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   369
                    tr.hookargs['node_last'] = hex(cl.node(clend - 1))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   370
                    hookargs = dict(tr.hookargs)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   371
                else:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   372
                    hookargs = dict(tr.hookargs)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   373
                    hookargs['node'] = hex(cl.node(clstart))
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   374
                    hookargs['node_last'] = hex(cl.node(clend - 1))
33671
38fc45721334 changegroup: wrap some ** expansions in strkwargs
Augie Fackler <augie@google.com>
parents: 33461
diff changeset
   375
                repo.hook('pretxnchangegroup',
38fc45721334 changegroup: wrap some ** expansions in strkwargs
Augie Fackler <augie@google.com>
parents: 33461
diff changeset
   376
                          throw=True, **pycompat.strkwargs(hookargs))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   377
38783
e7aa113b14f7 global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38556
diff changeset
   378
            added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
33456
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   379
            phaseall = None
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   380
            if srctype in ('push', 'serve'):
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   381
                # Old servers can not push the boundary themselves.
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   382
                # New servers won't push the boundary if changeset already
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   383
                # exists locally as secret
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   384
                #
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   385
                # We should not use added here but the list of all change in
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   386
                # the bundle
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   387
                if repo.publishing():
33456
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   388
                    targetphase = phaseall = phases.public
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   389
                else:
33456
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   390
                    # closer target phase computation
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   391
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   392
                    # Those changesets have been pushed from the
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   393
                    # outside, their phases are going to be pushed
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   394
                    # alongside. Therefor `targetphase` is
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   395
                    # ignored.
33456
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   396
                    targetphase = phaseall = phases.draft
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   397
            if added:
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   398
                phases.registernew(repo, tr, targetphase, added)
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   399
            if phaseall is not None:
ae052d04b89e phases: rework phase movement code in 'cg.apply' to use 'registernew'
Boris Feld <boris.feld@octobus.net>
parents: 33406
diff changeset
   400
                phases.advanceboundary(repo, tr, phaseall, cgnodes)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   401
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   402
            if changesets > 0:
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   403
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   404
                def runhooks():
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   405
                    # These hooks run when the lock releases, not when the
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   406
                    # transaction closes. So it's possible for the changelog
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   407
                    # to have changed since we last saw it.
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   408
                    if clstart >= len(repo):
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   409
                        return
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   410
33712
672ad4f3bb84 changegroup: more **kwargs
Augie Fackler <augie@google.com>
parents: 33671
diff changeset
   411
                    repo.hook("changegroup", **pycompat.strkwargs(hookargs))
27867
7ced54ebf972 with: use context manager for transaction in changegroup apply
Bryan O'Sullivan <bryano@fb.com>
parents: 27754
diff changeset
   412
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   413
                    for n in added:
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   414
                        args = hookargs.copy()
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   415
                        args['node'] = hex(n)
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   416
                        del args['node_last']
33712
672ad4f3bb84 changegroup: more **kwargs
Augie Fackler <augie@google.com>
parents: 33671
diff changeset
   417
                        repo.hook("incoming", **pycompat.strkwargs(args))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   418
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   419
                    newheads = [h for h in repo.heads()
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   420
                                if h not in oldheads]
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   421
                    repo.ui.log("incoming",
36737
2a5024109490 py3: fix int formatting of "incoming changes" log
Yuya Nishihara <yuya@tcha.org>
parents: 36465
diff changeset
   422
                                "%d incoming changes - new heads: %s\n",
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   423
                                len(added),
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   424
                                ', '.join([hex(c[:6]) for c in newheads]))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   425
32931
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   426
                tr.addpostclose('changegroup-runhooks-%020i' % clstart,
b08431e1b062 changegroup: delete "if True" and reflow
Martin von Zweigbergk <martinvonz@google.com>
parents: 32930
diff changeset
   427
                                lambda tr: repo._afterlock(runhooks))
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   428
        finally:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   429
            repo.ui.flush()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   430
        # never return 0 here:
32870
b441296f8e9c changegroup: rename "dh" to the clearer "deltaheads"
Martin von Zweigbergk <martinvonz@google.com>
parents: 32869
diff changeset
   431
        if deltaheads < 0:
33030
3e102a8dd52c bundle2: record changegroup data in 'op.records' (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 32931
diff changeset
   432
            ret = deltaheads - 1
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   433
        else:
33030
3e102a8dd52c bundle2: record changegroup data in 'op.records' (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 32931
diff changeset
   434
            ret = deltaheads + 1
33461
bb72031f0ea8 changegroup: stop returning and recording added nodes in 'cg.apply'
Boris Feld <boris.feld@octobus.net>
parents: 33456
diff changeset
   435
        return ret
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   436
34291
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   437
    def deltaiter(self):
34148
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   438
        """
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   439
        returns an iterator of the deltas in this changegroup
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   440
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   441
        Useful for passing to the underlying storage system to be stored.
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   442
        """
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   443
        chain = None
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   444
        for chunkdata in iter(lambda: self.deltachunk(chain), {}):
34294
05131c963767 changegroup: remove dictionary creation from deltachunk
Durham Goode <durham@fb.com>
parents: 34291
diff changeset
   445
            # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
05131c963767 changegroup: remove dictionary creation from deltachunk
Durham Goode <durham@fb.com>
parents: 34291
diff changeset
   446
            yield chunkdata
05131c963767 changegroup: remove dictionary creation from deltachunk
Durham Goode <durham@fb.com>
parents: 34291
diff changeset
   447
            chain = chunkdata[0]
34148
c8b6ed51386b changegroup: remove changegroup dependency from revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34103
diff changeset
   448
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   449
class cg2unpacker(cg1unpacker):
26708
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   450
    """Unpacker for cg2 streams.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   451
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   452
    cg2 streams add support for generaldelta, so the delta header
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   453
    format is slightly different. All other features about the data
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   454
    remain the same.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   455
    """
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   456
    deltaheader = _CHANGEGROUPV2_DELTA_HEADER
38896
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
   457
    deltaheadersize = deltaheader.size
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   458
    version = '02'
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   459
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   460
    def _deltaheader(self, headertuple, prevnode):
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   461
        node, p1, p2, deltabase, cs = headertuple
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   462
        flags = 0
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   463
        return node, p1, p2, deltabase, cs, flags
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   464
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   465
class cg3unpacker(cg2unpacker):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   466
    """Unpacker for cg3 streams.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   467
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   468
    cg3 streams add support for exchanging treemanifests and revlog
27753
d4071cc73f46 changegroup3: add empty chunk separating directories and files
Martin von Zweigbergk <martinvonz@google.com>
parents: 27752
diff changeset
   469
    flags. It adds the revlog flags to the delta header and an empty chunk
d4071cc73f46 changegroup3: add empty chunk separating directories and files
Martin von Zweigbergk <martinvonz@google.com>
parents: 27752
diff changeset
   470
    separating manifests and files.
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   471
    """
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   472
    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
38896
271854adc3a6 changegroup: make delta header struct formatters actual structs
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38895
diff changeset
   473
    deltaheadersize = deltaheader.size
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   474
    version = '03'
27920
da5f23362517 changegroup: cg3 has two empty groups *after* manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27867
diff changeset
   475
    _grouplistcount = 2 # One list of manifests and one list of files
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   476
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   477
    def _deltaheader(self, headertuple, prevnode):
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   478
        node, p1, p2, deltabase, cs, flags = headertuple
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   479
        return node, p1, p2, deltabase, cs, flags
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   480
38346
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   481
    def _unpackmanifests(self, repo, revmap, trp, prog):
83534c4ec58b changegroup: use progress helper in apply() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   482
        super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
29724
4e7be6e33269 changegroup: use `iter(callable, sentinel)` instead of while True
Augie Fackler <augie@google.com>
parents: 29690
diff changeset
   483
        for chunkdata in iter(self.filelogheader, {}):
27754
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
   484
            # If we get here, there are directory manifests in the changegroup
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
   485
            d = chunkdata["filename"]
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
   486
            repo.ui.debug("adding %s revisions\n" % d)
30339
6cdfb7e15a35 changegroup: remove remaining uses of repo.manifest
Durham Goode <durham@fb.com>
parents: 30294
diff changeset
   487
            dirlog = repo.manifestlog._revlog.dirlog(d)
34291
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   488
            deltas = self.deltaiter()
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
   489
            if not dirlog.addgroup(deltas, revmap, trp):
27754
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
   490
                raise error.Abort(_("received dir revlog group is empty"))
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
   491
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   492
class headerlessfixup(object):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   493
    def __init__(self, fh, h):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   494
        self._h = h
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   495
        self._fh = fh
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   496
    def read(self, n):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   497
        if self._h:
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   498
            d, self._h = self._h[:n], self._h[n:]
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   499
            if len(d) < n:
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
   500
                d += readexactly(self._fh, n - len(d))
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   501
            return d
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
   502
        return readexactly(self._fh, n)
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   503
38893
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   504
@attr.s(slots=True, frozen=True)
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   505
class revisiondelta(object):
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   506
    """Describes a delta entry in a changegroup.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   507
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   508
    Captured data is sufficient to serialize the delta into multiple
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   509
    formats.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   510
    """
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   511
    # 20 byte node of this revision.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   512
    node = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   513
    # 20 byte nodes of parent revisions.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   514
    p1node = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   515
    p2node = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   516
    # 20 byte node of node this delta is against.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   517
    basenode = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   518
    # 20 byte node of changeset revision this delta is associated with.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   519
    linknode = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   520
    # 2 bytes of flags to apply to revision data.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   521
    flags = attr.ib()
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   522
    # Iterable of chunks holding raw delta data.
23d582caae30 changegroup: capture revision delta in a data structure
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38892
diff changeset
   523
    deltachunks = attr.ib()
38883
ee1ea96cf9c9 changegroup: move ellipsisdata() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38835
diff changeset
   524
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   525
def _sortnodesnormal(store, nodes, reorder):
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   526
    """Sort nodes for changegroup generation and turn into revnums."""
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   527
    # for generaldelta revlogs, we linearize the revs; this will both be
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   528
    # much quicker and generate a much smaller bundle
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   529
    if (store._generaldelta and reorder is None) or reorder:
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   530
        dag = dagutil.revlogdag(store)
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   531
        return dag.linearize(set(store.rev(n) for n in nodes))
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   532
    else:
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   533
        return sorted([store.rev(n) for n in nodes])
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   534
38997
812eec3f89cb changegroup: remove _clnodetorev
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38996
diff changeset
   535
def _sortnodesellipsis(store, nodes, cl, lookup):
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   536
    """Sort nodes for changegroup generation and turn into revnums."""
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   537
    # Ellipses serving mode.
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   538
    #
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   539
    # In a perfect world, we'd generate better ellipsis-ified graphs
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   540
    # for non-changelog revlogs. In practice, we haven't started doing
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   541
    # that yet, so the resulting DAGs for the manifestlog and filelogs
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   542
    # are actually full of bogus parentage on all the ellipsis
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   543
    # nodes. This has the side effect that, while the contents are
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   544
    # correct, the individual DAGs might be completely out of whack in
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   545
    # a case like 882681bc3166 and its ancestors (back about 10
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   546
    # revisions or so) in the main hg repo.
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   547
    #
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   548
    # The one invariant we *know* holds is that the new (potentially
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   549
    # bogus) DAG shape will be valid if we order the nodes in the
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   550
    # order that they're introduced in dramatis personae by the
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   551
    # changelog, so what we do is we sort the non-changelog histories
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   552
    # by the order in which they are used by the changelog.
38997
812eec3f89cb changegroup: remove _clnodetorev
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38996
diff changeset
   553
    key = lambda n: cl.rev(lookup(n))
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   554
    return [store.rev(n) for n in sorted(nodes, key=key)]
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   555
38985
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   556
def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn):
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   557
    """Construct a revision delta for non-ellipses changegroup generation."""
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   558
    node = store.node(rev)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   559
    p1, p2 = store.parentrevs(rev)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   560
    base = deltaparentfn(store, rev, p1, p2, prev)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   561
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   562
    prefix = ''
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   563
    if store.iscensored(base) or store.iscensored(rev):
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   564
        try:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   565
            delta = store.revision(node, raw=True)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   566
        except error.CensoredNodeError as e:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   567
            delta = e.tombstone
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   568
        if base == nullrev:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   569
            prefix = mdiff.trivialdiffheader(len(delta))
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   570
        else:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   571
            baselen = store.rawsize(base)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   572
            prefix = mdiff.replacediffheader(baselen, len(delta))
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   573
    elif base == nullrev:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   574
        delta = store.revision(node, raw=True)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   575
        prefix = mdiff.trivialdiffheader(len(delta))
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   576
    else:
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   577
        delta = store.revdiff(base, rev)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   578
    p1n, p2n = store.parents(node)
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   579
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   580
    return revisiondelta(
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   581
        node=node,
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   582
        p1node=p1n,
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   583
        p2node=p2n,
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   584
        basenode=store.node(base),
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   585
        linknode=linknode,
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   586
        flags=store.flags(rev),
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   587
        deltachunks=(prefix, delta),
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   588
    )
d85b0d81112b changegroup: extract _revisiondeltanormal() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38984
diff changeset
   589
39004
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   590
def _revisiondeltanarrow(cl, store, ischangelog, rev, linkrev,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   591
                         linknode, clrevtolocalrev, fullclnodes,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   592
                         precomputedellipsis):
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   593
    linkparents = precomputedellipsis[linkrev]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   594
    def local(clrev):
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   595
        """Turn a changelog revnum into a local revnum.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   596
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   597
        The ellipsis dag is stored as revnums on the changelog,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   598
        but when we're producing ellipsis entries for
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   599
        non-changelog revlogs, we need to turn those numbers into
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   600
        something local. This does that for us, and during the
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   601
        changelog sending phase will also expand the stored
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   602
        mappings as needed.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   603
        """
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   604
        if clrev == nullrev:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   605
            return nullrev
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   606
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   607
        if ischangelog:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   608
            return clrev
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   609
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   610
        # Walk the ellipsis-ized changelog breadth-first looking for a
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   611
        # change that has been linked from the current revlog.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   612
        #
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   613
        # For a flat manifest revlog only a single step should be necessary
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   614
        # as all relevant changelog entries are relevant to the flat
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   615
        # manifest.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   616
        #
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   617
        # For a filelog or tree manifest dirlog however not every changelog
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   618
        # entry will have been relevant, so we need to skip some changelog
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   619
        # nodes even after ellipsis-izing.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   620
        walk = [clrev]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   621
        while walk:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   622
            p = walk[0]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   623
            walk = walk[1:]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   624
            if p in clrevtolocalrev:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   625
                return clrevtolocalrev[p]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   626
            elif p in fullclnodes:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   627
                walk.extend([pp for pp in cl.parentrevs(p)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   628
                                if pp != nullrev])
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   629
            elif p in precomputedellipsis:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   630
                walk.extend([pp for pp in precomputedellipsis[p]
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   631
                                if pp != nullrev])
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   632
            else:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   633
                # In this case, we've got an ellipsis with parents
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   634
                # outside the current bundle (likely an
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   635
                # incremental pull). We "know" that we can use the
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   636
                # value of this same revlog at whatever revision
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   637
                # is pointed to by linknode. "Know" is in scare
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   638
                # quotes because I haven't done enough examination
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   639
                # of edge cases to convince myself this is really
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   640
                # a fact - it works for all the (admittedly
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   641
                # thorough) cases in our testsuite, but I would be
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   642
                # somewhat unsurprised to find a case in the wild
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   643
                # where this breaks down a bit. That said, I don't
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   644
                # know if it would hurt anything.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   645
                for i in pycompat.xrange(rev, 0, -1):
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   646
                    if store.linkrev(i) == clrev:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   647
                        return i
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   648
                # We failed to resolve a parent for this node, so
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   649
                # we crash the changegroup construction.
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   650
                raise error.Abort(
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   651
                    'unable to resolve parent while packing %r %r'
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   652
                    ' for changeset %r' % (store.indexfile, rev, clrev))
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   653
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   654
        return nullrev
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   655
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   656
    if not linkparents or (
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   657
        store.parentrevs(rev) == (nullrev, nullrev)):
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   658
        p1, p2 = nullrev, nullrev
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   659
    elif len(linkparents) == 1:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   660
        p1, = sorted(local(p) for p in linkparents)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   661
        p2 = nullrev
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   662
    else:
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   663
        p1, p2 = sorted(local(p) for p in linkparents)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   664
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   665
    n = store.node(rev)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   666
    p1n, p2n = store.node(p1), store.node(p2)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   667
    flags = store.flags(rev)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   668
    flags |= revlog.REVIDX_ELLIPSIS
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   669
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   670
    # TODO: try and actually send deltas for ellipsis data blocks
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   671
    data = store.revision(n)
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   672
    diffheader = mdiff.trivialdiffheader(len(data))
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   673
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   674
    return revisiondelta(
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   675
        node=n,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   676
        p1node=p1n,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   677
        p2node=p2n,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   678
        basenode=nullid,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   679
        linknode=linknode,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   680
        flags=flags,
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   681
        deltachunks=(diffheader, data),
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   682
    )
e11d07cc125c changegroup: make _revisiondeltanarrow() a standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39003
diff changeset
   683
39009
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   684
def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   685
               deltaheaderfn, units=None,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   686
               ellipses=False, clrevtolocalrev=None, fullclnodes=None,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   687
               precomputedellipsis=None):
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   688
    """Calculate a delta group, yielding a sequence of changegroup chunks
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   689
    (strings).
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   690
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   691
    Given a list of changeset revs, return a set of deltas and
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   692
    metadata corresponding to nodes. The first delta is
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   693
    first parent(nodelist[0]) -> nodelist[0], the receiver is
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   694
    guaranteed to have this parent as it has all history before
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   695
    these changesets. In the case firstparent is nullrev the
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   696
    changegroup starts with a full revision.
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   697
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   698
    If units is not None, progress detail will be generated, units specifies
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   699
    the type of revlog that is touched (changelog, manifest, etc.).
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   700
    """
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   701
    # if we don't have any revisions touched by these changesets, bail
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   702
    if len(revs) == 0:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   703
        yield closechunk()
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   704
        return
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   705
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   706
    cl = repo.changelog
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   707
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   708
    # add the parent of the first rev
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   709
    p = store.parentrevs(revs[0])[0]
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   710
    revs.insert(0, p)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   711
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   712
    # build deltas
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   713
    progress = None
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   714
    if units is not None:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   715
        progress = repo.ui.makeprogress(_('bundling'), unit=units,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   716
                                        total=(len(revs) - 1))
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   717
    for r in pycompat.xrange(len(revs) - 1):
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   718
        if progress:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   719
            progress.update(r + 1)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   720
        prev, curr = revs[r], revs[r + 1]
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   721
        linknode = lookup(store.node(curr))
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   722
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   723
        if ellipses:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   724
            linkrev = cl.rev(linknode)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   725
            clrevtolocalrev[linkrev] = curr
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   726
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   727
            # This is a node to send in full, because the changeset it
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   728
            # corresponds to was a full changeset.
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   729
            if linknode in fullclnodes:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   730
                delta = _revisiondeltanormal(store, curr, prev, linknode,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   731
                                             deltaparentfn)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   732
            elif linkrev not in precomputedellipsis:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   733
                delta = None
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   734
            else:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   735
                delta = _revisiondeltanarrow(
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   736
                    cl, store, ischangelog, curr, linkrev, linknode,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   737
                    clrevtolocalrev, fullclnodes,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   738
                    precomputedellipsis)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   739
        else:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   740
            delta = _revisiondeltanormal(store, curr, prev, linknode,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   741
                                         deltaparentfn)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   742
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   743
        if not delta:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   744
            continue
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   745
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   746
        meta = deltaheaderfn(delta)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   747
        l = len(meta) + sum(len(x) for x in delta.deltachunks)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   748
        yield chunkheader(l)
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   749
        yield meta
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   750
        for x in delta.deltachunks:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   751
            yield x
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   752
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   753
    if progress:
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   754
        progress.complete()
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   755
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   756
    yield closechunk()
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   757
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
   758
class cgpacker(object):
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   759
    def __init__(self, repo, filematcher, version, allowreorder,
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
   760
                 deltaparentfn, builddeltaheader, manifestsend,
38925
8a13855c8dbe changegroup: always use the treemanifest-enabled version of _packmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 38910
diff changeset
   761
                 bundlecaps=None, ellipses=False,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
   762
                 shallow=False, ellipsisroots=None, fullnodes=None):
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   763
        """Given a source repo, construct a bundler.
32287
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   764
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   765
        filematcher is a matcher that matches on files to include in the
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   766
        changegroup. Used to facilitate sparse changegroups.
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   767
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   768
        allowreorder controls whether reordering of revisions is allowed.
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   769
        This value is used when ``bundle.reorder`` is ``auto`` or isn't
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   770
        set.
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   771
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
   772
        deltaparentfn is a callable that resolves the delta parent for
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
   773
        a specific revision.
38901
23ae0c07a3e1 changegroup: control delta parent behavior via constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38900
diff changeset
   774
38897
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
   775
        builddeltaheader is a callable that constructs the header for a group
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
   776
        delta.
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
   777
38898
67f37e8a5490 changegroup: pass end of manifests marker into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38897
diff changeset
   778
        manifestsend is a chunk to send after manifests have been fully emitted.
67f37e8a5490 changegroup: pass end of manifests marker into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38897
diff changeset
   779
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
   780
        ellipses indicates whether ellipsis serving mode is enabled.
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
   781
32287
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   782
        bundlecaps is optional and can be used to specify the set of
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   783
        capabilities which can be used to build the bundle. While bundlecaps is
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   784
        unused in core Mercurial, extensions rely on this feature to communicate
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   785
        capabilities to customize the changegroup packer.
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
   786
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
   787
        shallow indicates whether shallow data might be sent. The packer may
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
   788
        need to pack file contents not introduced by the changes being packed.
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
   789
38996
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   790
        fullnodes is the set of changelog nodes which should not be ellipsis
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   791
        nodes. We store this rather than the set of nodes that should be
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   792
        ellipsis because for very large histories we expect this to be
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   793
        significantly smaller.
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   794
        """
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   795
        assert filematcher
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   796
        self._filematcher = filematcher
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   797
38895
d7ac49c2353c changegroup: pass version into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38894
diff changeset
   798
        self.version = version
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
   799
        self._deltaparentfn = deltaparentfn
38897
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
   800
        self._builddeltaheader = builddeltaheader
38898
67f37e8a5490 changegroup: pass end of manifests marker into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38897
diff changeset
   801
        self._manifestsend = manifestsend
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
   802
        self._ellipses = ellipses
38895
d7ac49c2353c changegroup: pass version into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38894
diff changeset
   803
32287
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   804
        # Set of capabilities we can use to build the bundle.
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   805
        if bundlecaps is None:
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   806
            bundlecaps = set()
df3cf9422e1b changegroup: add bundlecaps back
Durham Goode <durham@fb.com>
parents: 32268
diff changeset
   807
        self._bundlecaps = bundlecaps
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
   808
        self._isshallow = shallow
38996
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   809
        self._fullclnodes = fullnodes
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   810
38907
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
   811
        # Maps ellipsis revs to their roots at the changelog level.
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
   812
        self._precomputedellipsis = ellipsisroots
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
   813
25831
578fc97904da generaldelta: mark experimental reordering option
Matt Mackall <mpm@selenic.com>
parents: 25823
diff changeset
   814
        # experimental config: bundle.reorder
33183
9f95f0bb343b configitems: register the 'bundle.reorder' config
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33036
diff changeset
   815
        reorder = repo.ui.config('bundle', 'reorder')
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   816
        if reorder == 'auto':
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   817
            self._reorder = allowreorder
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   818
        else:
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   819
            self._reorder = stringutil.parsebool(reorder)
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   820
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   821
        self._repo = repo
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
   822
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   823
        if self._repo.ui.verbose and not self._repo.ui.debugflag:
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   824
            self._verbosenote = self._repo.ui.note
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   825
        else:
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   826
            self._verbosenote = lambda s: None
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   827
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   828
    def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   829
        """Yield a sequence of changegroup byte chunks."""
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   830
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   831
        repo = self._repo
24978
f52560c64953 changegroup: drop _changelog and _manifest properties
Martin von Zweigbergk <martinvonz@google.com>
parents: 24977
diff changeset
   832
        cl = repo.changelog
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   833
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   834
        self._verbosenote(_('uncompressed size of bundle content:\n'))
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   835
        size = 0
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   836
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   837
        clstate, chunks = self._generatechangelog(cl, clnodes)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   838
        for chunk in chunks:
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   839
            size += len(chunk)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   840
            yield chunk
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   841
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   842
        self._verbosenote(_('%8.i (changelog)\n') % size)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   843
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   844
        clrevorder = clstate['clrevorder']
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   845
        mfs = clstate['mfs']
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   846
        changedfiles = clstate['changedfiles']
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   847
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   848
        # We need to make sure that the linkrev in the changegroup refers to
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   849
        # the first changeset that introduced the manifest or file revision.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   850
        # The fastpath is usually safer than the slowpath, because the filelogs
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   851
        # are walked in revlog order.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   852
        #
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   853
        # When taking the slowpath with reorder=None and the manifest revlog
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   854
        # uses generaldelta, the manifest may be walked in the "wrong" order.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   855
        # Without 'clrevorder', we would get an incorrect linkrev (see fix in
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   856
        # cc0ff93d0c0c).
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   857
        #
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   858
        # When taking the fastpath, we are only vulnerable to reordering
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   859
        # of the changelog itself. The changelog never uses generaldelta, so
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   860
        # it is only reordered when reorder=True. To handle this case, we
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   861
        # simply take the slowpath, which already has the 'clrevorder' logic.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   862
        # This was also fixed in cc0ff93d0c0c.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   863
        fastpathlinkrev = fastpathlinkrev and not self._reorder
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   864
        # Treemanifests don't work correctly with fastpathlinkrev
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   865
        # either, because we don't discover which directory nodes to
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   866
        # send along with files. This could probably be fixed.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   867
        fastpathlinkrev = fastpathlinkrev and (
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   868
            'treemanifest' not in repo.requirements)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   869
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   870
        fnodes = {}  # needed file nodes
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   871
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   872
        for chunk in self.generatemanifests(commonrevs, clrevorder,
39001
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
   873
                fastpathlinkrev, mfs, fnodes, source,
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
   874
                clstate['clrevtomanifestrev']):
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   875
            yield chunk
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   876
38983
fbbda9ff3deb changegroup: pass mfdicts properly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38982
diff changeset
   877
        mfdicts = None
fbbda9ff3deb changegroup: pass mfdicts properly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38982
diff changeset
   878
        if self._ellipses and self._isshallow:
fbbda9ff3deb changegroup: pass mfdicts properly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38982
diff changeset
   879
            mfdicts = [(self._repo.manifestlog[n].read(), lr)
fbbda9ff3deb changegroup: pass mfdicts properly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38982
diff changeset
   880
                       for (n, lr) in mfs.iteritems()]
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   881
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   882
        mfs.clear()
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   883
        clrevs = set(cl.rev(x) for x in clnodes)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   884
38999
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
   885
        for chunk in self.generatefiles(changedfiles, commonrevs,
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
   886
                                        source, mfdicts, fastpathlinkrev,
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
   887
                                        fnodes, clrevs):
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   888
            yield chunk
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   889
39002
eb8a0139ace3 changegroup: inline _close()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39001
diff changeset
   890
        yield closechunk()
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   891
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   892
        if clnodes:
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   893
            repo.hook('outgoing', node=hex(clnodes[0]), source=source)
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   894
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   895
    def _generatechangelog(self, cl, nodes):
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   896
        """Generate data for changelog chunks.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   897
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   898
        Returns a 2-tuple of a dict containing state and an iterable of
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   899
        byte chunks. The state will not be fully populated until the
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   900
        chunk stream has been fully consumed.
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   901
        """
23381
cc0ff93d0c0c changegroup: fix file linkrevs during reorders (issue4462)
Durham Goode <durham@fb.com>
parents: 23226
diff changeset
   902
        clrevorder = {}
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   903
        mfs = {} # needed manifests
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   904
        mfl = self._repo.manifestlog
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   905
        # TODO violates storage abstraction.
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   906
        mfrevlog = mfl._revlog
28241
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
   907
        changedfiles = set()
38998
40374b4a780f changegroup: track changelog to manifest revision map explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38997
diff changeset
   908
        clrevtomanifestrev = {}
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   909
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   910
        # Callback for the changelog, used to collect changed files and
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   911
        # manifest nodes.
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   912
        # Returns the linkrev node (identity in the changelog case).
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   913
        def lookupcl(x):
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   914
            c = cl.read(x)
23381
cc0ff93d0c0c changegroup: fix file linkrevs during reorders (issue4462)
Durham Goode <durham@fb.com>
parents: 23226
diff changeset
   915
            clrevorder[x] = len(clrevorder)
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   916
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
   917
            if self._ellipses:
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   918
                # Only update mfs if x is going to be sent. Otherwise we
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   919
                # end up with bogus linkrevs specified for manifests and
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   920
                # we skip some manifest nodes that we should otherwise
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   921
                # have sent.
38996
5baafb8fe253 changegroup: rename _fullnodes to _fullclnodes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38995
diff changeset
   922
                if (x in self._fullclnodes
38907
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
   923
                    or cl.rev(x) in self._precomputedellipsis):
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   924
                    n = c[0]
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   925
                    # Record the first changeset introducing this manifest
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   926
                    # version.
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   927
                    mfs.setdefault(n, x)
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   928
                    # Set this narrow-specific dict so we have the lowest
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   929
                    # manifest revnum to look up for this cl revnum. (Part of
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   930
                    # mapping changelog ellipsis parents to manifest ellipsis
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   931
                    # parents)
38998
40374b4a780f changegroup: track changelog to manifest revision map explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38997
diff changeset
   932
                    clrevtomanifestrev.setdefault(cl.rev(x), mfrevlog.rev(n))
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   933
                # We can't trust the changed files list in the changeset if the
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   934
                # client requested a shallow clone.
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
   935
                if self._isshallow:
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   936
                    changedfiles.update(mfl[c[0]].read().keys())
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   937
                else:
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   938
                    changedfiles.update(c[3])
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   939
            else:
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   940
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   941
                n = c[0]
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   942
                # record the first changeset introducing this manifest version
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   943
                mfs.setdefault(n, x)
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   944
                # Record a complete list of potentially-changed files in
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   945
                # this manifest.
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   946
                changedfiles.update(c[3])
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   947
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   948
            return x
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   949
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   950
        # Changelog doesn't benefit from reordering revisions. So send out
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   951
        # revisions in store order.
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   952
        revs = sorted(cl.rev(n) for n in nodes)
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
   953
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   954
        state = {
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   955
            'clrevorder': clrevorder,
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   956
            'mfs': mfs,
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   957
            'changedfiles': changedfiles,
38998
40374b4a780f changegroup: track changelog to manifest revision map explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38997
diff changeset
   958
            'clrevtomanifestrev': clrevtomanifestrev,
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   959
        }
38890
d706c77449f9 changegroup: move generate() modifications from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38889
diff changeset
   960
39009
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   961
        gen = deltagroup(
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   962
            self._repo, revs, cl, True, lookupcl,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   963
            self._deltaparentfn, self._builddeltaheader,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   964
            ellipses=self._ellipses,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   965
            units=_('changesets'),
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   966
            clrevtolocalrev={},
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   967
            fullclnodes=self._fullclnodes,
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
   968
            precomputedellipsis=self._precomputedellipsis)
28227
1c36cc8e7870 changegroup: extract generatemanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27953
diff changeset
   969
38976
f7228c907ef4 changegroup: factor changelog chunk generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38975
diff changeset
   970
        return state, gen
28227
1c36cc8e7870 changegroup: extract generatemanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27953
diff changeset
   971
1c36cc8e7870 changegroup: extract generatemanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27953
diff changeset
   972
    def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
39001
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
   973
                          fnodes, source, clrevtolocalrev):
34149
75cc1f1e11f2 changegroup: add source parameter to generatemanifests
Durham Goode <durham@fb.com>
parents: 34148
diff changeset
   974
        """Returns an iterator of changegroup chunks containing manifests.
75cc1f1e11f2 changegroup: add source parameter to generatemanifests
Durham Goode <durham@fb.com>
parents: 34148
diff changeset
   975
75cc1f1e11f2 changegroup: add source parameter to generatemanifests
Durham Goode <durham@fb.com>
parents: 34148
diff changeset
   976
        `source` is unused here, but is used by extensions like remotefilelog to
75cc1f1e11f2 changegroup: add source parameter to generatemanifests
Durham Goode <durham@fb.com>
parents: 34148
diff changeset
   977
        change what is sent based in pulls vs pushes, etc.
75cc1f1e11f2 changegroup: add source parameter to generatemanifests
Durham Goode <durham@fb.com>
parents: 34148
diff changeset
   978
        """
28227
1c36cc8e7870 changegroup: extract generatemanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27953
diff changeset
   979
        repo = self._repo
38997
812eec3f89cb changegroup: remove _clnodetorev
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38996
diff changeset
   980
        cl = repo.changelog
30294
bce79dfcf5e4 manifest: get rid of manifest.readshallowfast
Durham Goode <durham@fb.com>
parents: 30268
diff changeset
   981
        mfl = repo.manifestlog
bce79dfcf5e4 manifest: get rid of manifest.readshallowfast
Durham Goode <durham@fb.com>
parents: 30268
diff changeset
   982
        dirlog = mfl._revlog.dirlog
28232
829d369fc5a8 changegroup: write root manifests and subdir manifests in a single loop
Martin von Zweigbergk <martinvonz@google.com>
parents: 28231
diff changeset
   983
        tmfnodes = {'': mfs}
28227
1c36cc8e7870 changegroup: extract generatemanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27953
diff changeset
   984
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   985
        # Callback for the manifest, used to collect linkrevs for filelog
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   986
        # revisions.
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   987
        # Returns the linkrev node (collected in lookupcl).
35012
d80380ba8e7d changegroup: use any node, not min(), in treemanifest's generatemanifests
Kyle Lippincott <spectral@google.com>
parents: 34734
diff changeset
   988
        def makelookupmflinknode(dir, nodes):
28231
3faba927dd93 changegroup: introduce makelookupmflinknode(dir)
Martin von Zweigbergk <martinvonz@google.com>
parents: 28230
diff changeset
   989
            if fastpathlinkrev:
3faba927dd93 changegroup: introduce makelookupmflinknode(dir)
Martin von Zweigbergk <martinvonz@google.com>
parents: 28230
diff changeset
   990
                assert not dir
3faba927dd93 changegroup: introduce makelookupmflinknode(dir)
Martin von Zweigbergk <martinvonz@google.com>
parents: 28230
diff changeset
   991
                return mfs.__getitem__
3faba927dd93 changegroup: introduce makelookupmflinknode(dir)
Martin von Zweigbergk <martinvonz@google.com>
parents: 28230
diff changeset
   992
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   993
            def lookupmflinknode(x):
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   994
                """Callback for looking up the linknode for manifests.
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
   995
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   996
                Returns the linkrev node for the specified manifest.
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
   997
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   998
                SIDE EFFECT:
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   999
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1000
                1) fclnodes gets populated with the list of relevant
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1001
                   file nodes if we're not using fastpathlinkrev
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1002
                2) When treemanifests are in use, collects treemanifest nodes
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1003
                   to send
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
  1004
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1005
                Note that this means manifests must be completely sent to
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1006
                the client before you can trust the list of files and
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1007
                treemanifests to send.
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
  1008
                """
35012
d80380ba8e7d changegroup: use any node, not min(), in treemanifest's generatemanifests
Kyle Lippincott <spectral@google.com>
parents: 34734
diff changeset
  1009
                clnode = nodes[x]
30294
bce79dfcf5e4 manifest: get rid of manifest.readshallowfast
Durham Goode <durham@fb.com>
parents: 30268
diff changeset
  1010
                mdata = mfl.get(dir, x).readfast(shallow=True)
28241
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1011
                for p, n, fl in mdata.iterentries():
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1012
                    if fl == 't': # subdirectory manifest
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1013
                        subdir = dir + p + '/'
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1014
                        tmfclnodes = tmfnodes.setdefault(subdir, {})
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1015
                        tmfclnode = tmfclnodes.setdefault(n, clnode)
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1016
                        if clrevorder[clnode] < clrevorder[tmfclnode]:
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1017
                            tmfclnodes[n] = clnode
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1018
                    else:
a4286175ecba changegroup: drop special-casing of flat manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28240
diff changeset
  1019
                        f = dir + p
28240
1ac8ce137377 changegroup: fix treemanifests on merges
Martin von Zweigbergk <martinvonz@google.com>
parents: 28232
diff changeset
  1020
                        fclnodes = fnodes.setdefault(f, {})
1ac8ce137377 changegroup: fix treemanifests on merges
Martin von Zweigbergk <martinvonz@google.com>
parents: 28232
diff changeset
  1021
                        fclnode = fclnodes.setdefault(n, clnode)
1ac8ce137377 changegroup: fix treemanifests on merges
Martin von Zweigbergk <martinvonz@google.com>
parents: 28232
diff changeset
  1022
                        if clrevorder[clnode] < clrevorder[fclnode]:
1ac8ce137377 changegroup: fix treemanifests on merges
Martin von Zweigbergk <martinvonz@google.com>
parents: 28232
diff changeset
  1023
                            fclnodes[n] = clnode
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
  1024
                return clnode
28231
3faba927dd93 changegroup: introduce makelookupmflinknode(dir)
Martin von Zweigbergk <martinvonz@google.com>
parents: 28230
diff changeset
  1025
            return lookupmflinknode
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
  1026
28228
abf120262683 changegroup: make _packmanifests() dumber
Martin von Zweigbergk <martinvonz@google.com>
parents: 28227
diff changeset
  1027
        size = 0
28232
829d369fc5a8 changegroup: write root manifests and subdir manifests in a single loop
Martin von Zweigbergk <martinvonz@google.com>
parents: 28231
diff changeset
  1028
        while tmfnodes:
35012
d80380ba8e7d changegroup: use any node, not min(), in treemanifest's generatemanifests
Kyle Lippincott <spectral@google.com>
parents: 34734
diff changeset
  1029
            dir, nodes = tmfnodes.popitem()
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1030
            store = dirlog(dir)
39007
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1031
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1032
            if not self._filematcher.visitdir(store._dir[:-1] or '.'):
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1033
                prunednodes = []
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1034
            else:
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1035
                frev, flr = store.rev, store.linkrev
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1036
                prunednodes = [n for n in nodes
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1037
                               if flr(frev(n)) not in commonrevs]
39005
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1038
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1039
            if dir and not prunednodes:
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1040
                continue
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1041
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1042
            lookupfn = makelookupmflinknode(dir, nodes)
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1043
39005
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1044
            if self._ellipses:
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1045
                revs = _sortnodesellipsis(store, prunednodes, cl,
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1046
                                          lookupfn)
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1047
            else:
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1048
                revs = _sortnodesnormal(store, prunednodes,
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1049
                                        self._reorder)
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1050
39006
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1051
            if dir:
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1052
                assert self.version == b'03'
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1053
                chunk = _fileheader(dir)
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1054
                size += len(chunk)
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1055
                yield chunk
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1056
39009
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
  1057
            it = deltagroup(
39008
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1058
                self._repo, revs, store, False, lookupfn,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1059
                self._deltaparentfn, self._builddeltaheader,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1060
                ellipses=self._ellipses,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1061
                units=_('manifests'),
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1062
                clrevtolocalrev=clrevtolocalrev,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1063
                fullclnodes=self._fullclnodes,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1064
                precomputedellipsis=self._precomputedellipsis)
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1065
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1066
            for chunk in it:
39006
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1067
                size += len(chunk)
5959ef78d834 changegroup: inline _packmanifests() into generatemanifests()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39005
diff changeset
  1068
                yield chunk
39005
d56a6b78de3b changegroup: invert conditional and dedent
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39004
diff changeset
  1069
28229
8e13b2379407 changegroup: include subdirectory manifests in verbose size
Martin von Zweigbergk <martinvonz@google.com>
parents: 28228
diff changeset
  1070
        self._verbosenote(_('%8.i (manifests)\n') % size)
38898
67f37e8a5490 changegroup: pass end of manifests marker into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38897
diff changeset
  1071
        yield self._manifestsend
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
  1072
24897
5c35a6040352 changegroup: document that 'source' parameter exists for extensions
Martin von Zweigbergk <martinvonz@google.com>
parents: 24896
diff changeset
  1073
    # The 'source' parameter is useful for extensions
38999
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1074
    def generatefiles(self, changedfiles, commonrevs, source,
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1075
                      mfdicts, fastpathlinkrev, fnodes, clrevs):
38889
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1076
        changedfiles = list(filter(self._filematcher, changedfiles))
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1077
38999
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1078
        if not fastpathlinkrev:
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1079
            def normallinknodes(unused, fname):
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1080
                return fnodes.get(fname, {})
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1081
        else:
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1082
            cln = self._repo.changelog.node
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1083
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1084
            def normallinknodes(store, fname):
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1085
                flinkrev = store.linkrev
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1086
                fnode = store.node
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1087
                revs = ((r, flinkrev(r)) for r in store)
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1088
                return dict((fnode(r), cln(lr))
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1089
                            for r, lr in revs if lr in clrevs)
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1090
39001
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
  1091
        clrevtolocalrev = {}
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
  1092
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
  1093
        if self._isshallow:
38889
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1094
            # In a shallow clone, the linknodes callback needs to also include
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1095
            # those file nodes that are in the manifests we sent but weren't
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1096
            # introduced by those manifests.
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1097
            commonctxs = [self._repo[c] for c in commonrevs]
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1098
            clrev = self._repo.changelog.rev
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1099
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1100
            # Defining this function has a side-effect of overriding the
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1101
            # function of the same name that was passed in as an argument.
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1102
            # TODO have caller pass in appropriate function.
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1103
            def linknodes(flog, fname):
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1104
                for c in commonctxs:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1105
                    try:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1106
                        fnode = c.filenode(fname)
39001
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
  1107
                        clrevtolocalrev[c.rev()] = flog.rev(fnode)
38889
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1108
                    except error.ManifestLookupError:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1109
                        pass
38999
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1110
                links = normallinknodes(flog, fname)
38889
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1111
                if len(links) != len(mfdicts):
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1112
                    for mf, lr in mfdicts:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1113
                        fnode = mf.get(fname, None)
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1114
                        if fnode in links:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1115
                            links[fnode] = min(links[fnode], lr, key=clrev)
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1116
                        elif fnode:
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1117
                            links[fnode] = lr
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1118
                return links
38999
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1119
        else:
b83e9c503f2f changegroup: define linknodes callbacks in generatefiles()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38998
diff changeset
  1120
            linknodes = normallinknodes
38889
a06aab274aef changegroup: move generatefiles() from narrow
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38888
diff changeset
  1121
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1122
        repo = self._repo
38997
812eec3f89cb changegroup: remove _clnodetorev
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38996
diff changeset
  1123
        cl = repo.changelog
38410
1c5c4a5dd86d changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38382
diff changeset
  1124
        progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
1c5c4a5dd86d changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38382
diff changeset
  1125
                                        total=len(changedfiles))
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1126
        for i, fname in enumerate(sorted(changedfiles)):
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1127
            filerevlog = repo.file(fname)
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1128
            if not filerevlog:
37339
5859800edfc5 changegroup: remove "revlog" from error message
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37132
diff changeset
  1129
                raise error.Abort(_("empty or missing file data for %s") %
5859800edfc5 changegroup: remove "revlog" from error message
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37132
diff changeset
  1130
                                  fname)
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1131
39001
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
  1132
            clrevtolocalrev.clear()
a6e1ff40e335 changegroup: pass clrevtolocalrev to each group
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39000
diff changeset
  1133
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
  1134
            linkrevnodes = linknodes(filerevlog, fname)
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
  1135
            # Lookup for filenodes, we collected the linkrev nodes above in the
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
  1136
            # fastpath case and with lookupmf in the slowpath case.
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
  1137
            def lookupfilelog(x):
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
  1138
                return linkrevnodes[x]
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
  1139
39007
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1140
            frev, flr = filerevlog.rev, filerevlog.linkrev
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1141
            filenodes = [n for n in linkrevnodes
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1142
                         if flr(frev(n)) not in commonrevs]
39f5c7afdc25 changegroup: inline _prune() into call sites
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39006
diff changeset
  1143
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
  1144
            if filenodes:
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1145
                if self._ellipses:
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1146
                    revs = _sortnodesellipsis(filerevlog, filenodes,
38997
812eec3f89cb changegroup: remove _clnodetorev
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38996
diff changeset
  1147
                                              cl, lookupfilelog)
38982
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1148
                else:
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1149
                    revs = _sortnodesnormal(filerevlog, filenodes,
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1150
                                            self._reorder)
037debbf869c changegroup: pass sorted revisions into group() (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38981
diff changeset
  1151
38410
1c5c4a5dd86d changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38382
diff changeset
  1152
                progress.update(i + 1, item=fname)
38981
227ebd88ce5e changegroup: pull _fileheader out of cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38980
diff changeset
  1153
                h = _fileheader(fname)
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
  1154
                size = len(h)
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
  1155
                yield h
39008
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1156
39009
9e8eb2b444e5 changegroup: extract cgpacker.group() to standalone function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39008
diff changeset
  1157
                it = deltagroup(
39008
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1158
                    self._repo, revs, filerevlog, False, lookupfilelog,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1159
                    self._deltaparentfn, self._builddeltaheader,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1160
                    ellipses=self._ellipses,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1161
                    clrevtolocalrev=clrevtolocalrev,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1162
                    fullclnodes=self._fullclnodes,
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1163
                    precomputedellipsis=self._precomputedellipsis)
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1164
8c84f1ef949e changegroup: pass all state into group()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39007
diff changeset
  1165
                for chunk in it:
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
  1166
                    size += len(chunk)
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
  1167
                    yield chunk
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
  1168
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
38410
1c5c4a5dd86d changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38382
diff changeset
  1169
        progress.complete()
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
  1170
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1171
def _deltaparentprev(store, rev, p1, p2, prev):
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1172
    """Resolve a delta parent to the previous revision.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1173
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1174
    Used for version 1 changegroups, which don't support generaldelta.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1175
    """
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1176
    return prev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1177
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1178
def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1179
    """Resolve a delta parent when general deltas are supported."""
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1180
    dp = store.deltaparent(rev)
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1181
    if dp == nullrev and store.storedeltachains:
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1182
        # Avoid sending full revisions when delta parent is null. Pick prev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1183
        # in that case. It's tempting to pick p1 in this case, as p1 will
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1184
        # be smaller in the common case. However, computing a delta against
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1185
        # p1 may require resolving the raw text of p1, which could be
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1186
        # expensive. The revlog caches should have prev cached, meaning
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1187
        # less CPU for changegroup generation. There is likely room to add
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1188
        # a flag and/or config option to control this behavior.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1189
        base = prev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1190
    elif dp == nullrev:
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1191
        # revlog is configured to use full snapshot for a reason,
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1192
        # stick to full snapshot.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1193
        base = nullrev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1194
    elif dp not in (p1, p2, prev):
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1195
        # Pick prev when we can't be sure remote has the base revision.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1196
        return prev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1197
    else:
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1198
        base = dp
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1199
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1200
    if base != nullrev and not store.candelta(base, rev):
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1201
        base = nullrev
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1202
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1203
    return base
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1204
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1205
def _deltaparentellipses(store, rev, p1, p2, prev):
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1206
    """Resolve a delta parent when in ellipses mode."""
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1207
    # TODO: send better deltas when in narrow mode.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1208
    #
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1209
    # changegroup.group() loops over revisions to send,
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1210
    # including revisions we'll skip. What this means is that
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1211
    # `prev` will be a potentially useless delta base for all
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1212
    # ellipsis nodes, as the client likely won't have it. In
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1213
    # the future we should do bookkeeping about which nodes
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1214
    # have been sent to the client, and try to be
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1215
    # significantly smarter about delta bases. This is
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1216
    # slightly tricky because this same code has to work for
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1217
    # all revlogs, and we don't have the linkrev/linknode here.
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1218
    return p1
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1219
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1220
def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1221
                   shallow=False, ellipsisroots=None, fullnodes=None):
38897
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1222
    builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1223
        d.node, d.p1node, d.p2node, d.linknode)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1224
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1225
    return cgpacker(repo, filematcher, b'01',
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1226
                    deltaparentfn=_deltaparentprev,
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1227
                    allowreorder=None,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1228
                    builddeltaheader=builddeltaheader,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1229
                    manifestsend=b'',
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
  1230
                    bundlecaps=bundlecaps,
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1231
                    ellipses=ellipses,
38907
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
  1232
                    shallow=shallow,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1233
                    ellipsisroots=ellipsisroots,
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1234
                    fullnodes=fullnodes)
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1235
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1236
def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1237
                   shallow=False, ellipsisroots=None, fullnodes=None):
38897
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1238
    builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1239
        d.node, d.p1node, d.p2node, d.basenode, d.linknode)
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1240
38900
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
  1241
    # Since generaldelta is directly supported by cg2, reordering
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
  1242
    # generally doesn't help, so we disable it by default (treating
6e999a2d8fe7 changegroup: control reordering via constructor argument
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38899
diff changeset
  1243
    # bundle.reorder=auto just like bundle.reorder=False).
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1244
    return cgpacker(repo, filematcher, b'02',
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1245
                    deltaparentfn=_deltaparentgeneraldelta,
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1246
                    allowreorder=False,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1247
                    builddeltaheader=builddeltaheader,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1248
                    manifestsend=b'',
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
  1249
                    bundlecaps=bundlecaps,
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1250
                    ellipses=ellipses,
38907
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
  1251
                    shallow=shallow,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1252
                    ellipsisroots=ellipsisroots,
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1253
                    fullnodes=fullnodes)
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1254
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1255
def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1256
                   shallow=False, ellipsisroots=None, fullnodes=None):
38897
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1257
    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1258
        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
bd64b8b8f0dd changegroup: pass function to build delta header into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38896
diff changeset
  1259
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1260
    deltaparentfn = (_deltaparentellipses if ellipses
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1261
                     else _deltaparentgeneraldelta)
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1262
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1263
    return cgpacker(repo, filematcher, b'03',
38975
87b737b78bd0 changegroup: pass function to resolve delta parents into constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38974
diff changeset
  1264
                    deltaparentfn=deltaparentfn,
38902
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1265
                    allowreorder=False,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1266
                    builddeltaheader=builddeltaheader,
4c99c6d1ef95 changegroup: rename cg1packer to cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38901
diff changeset
  1267
                    manifestsend=closechunk(),
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
  1268
                    bundlecaps=bundlecaps,
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1269
                    ellipses=ellipses,
38907
ad4c4cc9a5ac changegroup: pass ellipsis roots into cgpacker constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38906
diff changeset
  1270
                    shallow=shallow,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1271
                    ellipsisroots=ellipsisroots,
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1272
                    fullnodes=fullnodes)
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1273
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1274
_packermap = {'01': (_makecg1packer, cg1unpacker),
26709
42733e956887 changegroup: reformat packermap and add comment
Augie Fackler <augie@google.com>
parents: 26708
diff changeset
  1275
             # cg2 adds support for exchanging generaldelta
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1276
             '02': (_makecg2packer, cg2unpacker),
27753
d4071cc73f46 changegroup3: add empty chunk separating directories and files
Martin von Zweigbergk <martinvonz@google.com>
parents: 27752
diff changeset
  1277
             # cg3 adds support for exchanging revlog flags and treemanifests
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1278
             '03': (_makecg3packer, cg3unpacker),
26709
42733e956887 changegroup: reformat packermap and add comment
Augie Fackler <augie@google.com>
parents: 26708
diff changeset
  1279
}
23168
a92ba36a1a9d changegroup: add a "packermap" dictionary to track different packer versions
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 22971
diff changeset
  1280
30627
7ace5304fec5 changegroup: pass 'repo' to allsupportedversions
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30626
diff changeset
  1281
def allsupportedversions(repo):
27928
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1282
    versions = set(_packermap.keys())
30627
7ace5304fec5 changegroup: pass 'repo' to allsupportedversions
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30626
diff changeset
  1283
    if not (repo.ui.configbool('experimental', 'changegroup3') or
30628
a001cd7296a5 changegroup: simplify logic around enabling changegroup 03
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30627
diff changeset
  1284
            repo.ui.configbool('experimental', 'treemanifest') or
a001cd7296a5 changegroup: simplify logic around enabling changegroup 03
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30627
diff changeset
  1285
            'treemanifest' in repo.requirements):
30626
438532c99b54 changegroup: simplify 'allsupportedversions' logic
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30589
diff changeset
  1286
        versions.discard('03')
27953
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1287
    return versions
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1288
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1289
# Changegroup versions that can be applied to the repo
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1290
def supportedincomingversions(repo):
30628
a001cd7296a5 changegroup: simplify logic around enabling changegroup 03
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30627
diff changeset
  1291
    return allsupportedversions(repo)
27953
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1292
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1293
# Changegroup versions that can be created from the repo
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1294
def supportedoutgoingversions(repo):
30627
7ace5304fec5 changegroup: pass 'repo' to allsupportedversions
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 30626
diff changeset
  1295
    versions = allsupportedversions(repo)
27953
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1296
    if 'treemanifest' in repo.requirements:
27928
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1297
        # Versions 01 and 02 support only flat manifests and it's just too
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1298
        # expensive to convert between the flat manifest and tree manifest on
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1299
        # the fly. Since tree manifests are hashed differently, all of history
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1300
        # would have to be converted. Instead, we simply don't even pretend to
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1301
        # support versions 01 and 02.
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1302
        versions.discard('01')
c0f11347b107 changegroup: don't support versions 01 and 02 with treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27920
diff changeset
  1303
        versions.discard('02')
38835
a232e6744ba3 narrow: move requirement constant from changegroup to repository
Martin von Zweigbergk <martinvonz@google.com>
parents: 38806
diff changeset
  1304
    if repository.NARROW_REQUIREMENT in repo.requirements:
36465
94709406f10d narrow: move changegroup.supportedoutgoingversions() override to core
Martin von Zweigbergk <martinvonz@google.com>
parents: 36464
diff changeset
  1305
        # Versions 01 and 02 don't support revlog flags, and we need to
94709406f10d narrow: move changegroup.supportedoutgoingversions() override to core
Martin von Zweigbergk <martinvonz@google.com>
parents: 36464
diff changeset
  1306
        # support that for stripping and unbundling to work.
94709406f10d narrow: move changegroup.supportedoutgoingversions() override to core
Martin von Zweigbergk <martinvonz@google.com>
parents: 36464
diff changeset
  1307
        versions.discard('01')
94709406f10d narrow: move changegroup.supportedoutgoingversions() override to core
Martin von Zweigbergk <martinvonz@google.com>
parents: 36464
diff changeset
  1308
        versions.discard('02')
37132
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1309
    if LFS_REQUIREMENT in repo.requirements:
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1310
        # Versions 01 and 02 don't support revlog flags, and we need to
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1311
        # mark LFS entries with REVIDX_EXTSTORED.
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1312
        versions.discard('01')
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1313
        versions.discard('02')
a54113fcc8c9 lfs: move the 'supportedoutgoingversions' handling to changegroup.py
Matt Harbison <matt_harbison@yahoo.com>
parents: 37084
diff changeset
  1314
27752
29cfc474c5fd changegroup3: introduce experimental.changegroup3 boolean config
Martin von Zweigbergk <martinvonz@google.com>
parents: 27751
diff changeset
  1315
    return versions
27751
a40e2f7fe49d changegroup: hide packermap behind methods
Martin von Zweigbergk <martinvonz@google.com>
parents: 27739
diff changeset
  1316
34144
91f0677dc920 repair: preserve phase also when not using generaldelta (issue5678)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33461
diff changeset
  1317
def localversion(repo):
91f0677dc920 repair: preserve phase also when not using generaldelta (issue5678)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33461
diff changeset
  1318
    # Finds the best version to use for bundles that are meant to be used
91f0677dc920 repair: preserve phase also when not using generaldelta (issue5678)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33461
diff changeset
  1319
    # locally, such as those from strip and shelve, and temporary bundles.
91f0677dc920 repair: preserve phase also when not using generaldelta (issue5678)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33461
diff changeset
  1320
    return max(supportedoutgoingversions(repo))
91f0677dc920 repair: preserve phase also when not using generaldelta (issue5678)
Martin von Zweigbergk <martinvonz@google.com>
parents: 33461
diff changeset
  1321
27929
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1322
def safeversion(repo):
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1323
    # Finds the smallest version that it's safe to assume clients of the repo
27931
1289a122cf3f shelve: use cg3 for treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27929
diff changeset
  1324
    # will support. For example, all hg versions that support generaldelta also
1289a122cf3f shelve: use cg3 for treemanifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 27929
diff changeset
  1325
    # support changegroup 02.
27953
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1326
    versions = supportedoutgoingversions(repo)
27929
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1327
    if 'generaldelta' in repo.requirements:
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1328
        versions.discard('01')
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1329
    assert versions
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1330
    return min(versions)
3b2ac2115464 changegroup: introduce safeversion()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27928
diff changeset
  1331
38904
cdb9bc216771 changegroup: declare shallow flag in constructor
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38903
diff changeset
  1332
def getbundler(version, repo, bundlecaps=None, filematcher=None,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1333
               ellipses=False, shallow=False, ellipsisroots=None,
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1334
               fullnodes=None):
27953
88609cfa3745 changegroup: fix pulling to treemanifest repo from flat repo (issue5066)
Martin von Zweigbergk <martinvonz@google.com>
parents: 27946
diff changeset
  1335
    assert version in supportedoutgoingversions(repo)
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1336
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1337
    if filematcher is None:
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1338
        filematcher = matchmod.alwaysmatcher(repo.root, '')
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1339
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1340
    if version == '01' and not filematcher.always():
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1341
        raise error.ProgrammingError('version 01 changegroups do not support '
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1342
                                     'sparse file matchers')
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1343
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1344
    if ellipses and version in (b'01', b'02'):
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1345
        raise error.Abort(
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1346
            _('ellipsis nodes require at least cg3 on client and server, '
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1347
              'but negotiated version %s') % version)
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1348
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1349
    # Requested files could include files not in the local store. So
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1350
    # filter those out.
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1351
    filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1352
                                             filematcher)
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1353
38894
19344024a8e1 changegroup: define functions for creating changegroup packers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38893
diff changeset
  1354
    fn = _packermap[version][0]
38908
1469584ad5fe changegroup: specify ellipses mode explicitly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38907
diff changeset
  1355
    return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
38909
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1356
              shallow=shallow, ellipsisroots=ellipsisroots,
1af339c22aeb changegroup: move fullnodes into cgpacker
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38908
diff changeset
  1357
              fullnodes=fullnodes)
27751
a40e2f7fe49d changegroup: hide packermap behind methods
Martin von Zweigbergk <martinvonz@google.com>
parents: 27739
diff changeset
  1358
29593
953839de96ab bundle2: store changeset count when creating file bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29371
diff changeset
  1359
def getunbundler(version, fh, alg, extras=None):
953839de96ab bundle2: store changeset count when creating file bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29371
diff changeset
  1360
    return _packermap[version][1](fh, alg, extras=extras)
27751
a40e2f7fe49d changegroup: hide packermap behind methods
Martin von Zweigbergk <martinvonz@google.com>
parents: 27739
diff changeset
  1361
20926
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1362
def _changegroupinfo(repo, nodes, source):
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1363
    if repo.ui.verbose or source == 'bundle':
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1364
        repo.ui.status(_("%d changesets found\n") % len(nodes))
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1365
    if repo.ui.debugflag:
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1366
        repo.ui.debug("list of changesets:\n")
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1367
        for node in nodes:
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1368
            repo.ui.debug("%s\n" % hex(node))
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1369
34096
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1370
def makechangegroup(repo, outgoing, version, source, fastpath=False,
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1371
                    bundlecaps=None):
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1372
    cgstream = makestream(repo, outgoing, version, source,
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1373
                          fastpath=fastpath, bundlecaps=bundlecaps)
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1374
    return getunbundler(version, util.chunkbuffer(cgstream), None,
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1375
                        {'clcount': len(outgoing.missing) })
f85dfde1731a changegroup: replace getsubset with makechangegroup
Durham Goode <durham@fb.com>
parents: 34091
diff changeset
  1376
34103
92f1e2be8ab6 changegroup: rename getsubsetraw to makestream
Durham Goode <durham@fb.com>
parents: 34101
diff changeset
  1377
def makestream(repo, outgoing, version, source, fastpath=False,
38794
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1378
               bundlecaps=None, filematcher=None):
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1379
    bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1d01cf0416a5 changegroup: move file matcher from narrow extension
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
  1380
                         filematcher=filematcher)
34103
92f1e2be8ab6 changegroup: rename getsubsetraw to makestream
Durham Goode <durham@fb.com>
parents: 34101
diff changeset
  1381
20925
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1382
    repo = repo.unfiltered()
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1383
    commonrevs = outgoing.common
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1384
    csets = outgoing.missing
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1385
    heads = outgoing.missingheads
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1386
    # We go through the fast path if we get told to, or if all (unfiltered
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1387
    # heads have been requested (since we then know there all linkrevs will
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1388
    # be pulled by the client).
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1389
    heads.sort()
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1390
    fastpathlinkrev = fastpath or (
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1391
            repo.filtername is None and heads == sorted(repo.heads()))
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1392
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
  1393
    repo.hook('preoutgoing', throw=True, source=source)
20926
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
  1394
    _changegroupinfo(repo, csets, source)
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
  1395
    return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
  1396
28361
277a22cd8741 changegroup: progress for added files is not measured in "chunks"
Martin von Zweigbergk <martinvonz@google.com>
parents: 28360
diff changeset
  1397
def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1398
    revisions = 0
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1399
    files = 0
38382
daa08d45740f changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
  1400
    progress = repo.ui.makeprogress(_('files'), unit=_('files'),
daa08d45740f changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
  1401
                                    total=expectedfiles)
29724
4e7be6e33269 changegroup: use `iter(callable, sentinel)` instead of while True
Augie Fackler <augie@google.com>
parents: 29690
diff changeset
  1402
    for chunkdata in iter(source.filelogheader, {}):
28361
277a22cd8741 changegroup: progress for added files is not measured in "chunks"
Martin von Zweigbergk <martinvonz@google.com>
parents: 28360
diff changeset
  1403
        files += 1
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1404
        f = chunkdata["filename"]
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1405
        repo.ui.debug("adding %s revisions\n" % f)
38382
daa08d45740f changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
  1406
        progress.increment()
27754
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
  1407
        fl = repo.file(f)
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1408
        o = len(fl)
24120
a450e0a2ba0a revlog: in addgroup, reject ill-formed deltas based on censored nodes
Mike Edgar <adgar@google.com>
parents: 23897
diff changeset
  1409
        try:
34291
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
  1410
            deltas = source.deltaiter()
1db9abf407c5 revlog: add revmap back to revlog.addgroup
Durham Goode <durham@fb.com>
parents: 34257
diff changeset
  1411
            if not fl.addgroup(deltas, revmap, trp):
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1412
                raise error.Abort(_("received file revlog group is empty"))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25624
diff changeset
  1413
        except error.CensoredBaseError as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1414
            raise error.Abort(_("received delta base is censored: %s") % e)
27754
a09f143daaf4 changegroup3: move treemanifest support into _unpackmanifests()
Martin von Zweigbergk <martinvonz@google.com>
parents: 27753
diff changeset
  1415
        revisions += len(fl) - o
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1416
        if f in needfiles:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1417
            needs = needfiles[f]
38783
e7aa113b14f7 global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38556
diff changeset
  1418
            for new in pycompat.xrange(o, len(fl)):
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1419
                n = fl.node(new)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1420
                if n in needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1421
                    needs.remove(n)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1422
                else:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1423
                    raise error.Abort(
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1424
                        _("received spurious file revlog entry"))
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1425
            if not needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1426
                del needfiles[f]
38382
daa08d45740f changegroup: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
  1427
    progress.complete()
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1428
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1429
    for f, needs in needfiles.iteritems():
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1430
        fl = repo.file(f)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1431
        for n in needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1432
            try:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1433
                fl.rev(n)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1434
            except error.LookupError:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1435
                raise error.Abort(
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1436
                    _('missing file data for %s:%s - run hg verify') %
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1437
                    (f, hex(n)))
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1438
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1439
    return revisions, files