mercurial/changegroup.py
author Martin von Zweigbergk <martinvonz@google.com>
Fri, 08 Jan 2016 21:14:08 -0800
changeset 27735 bd37212c20ed
parent 27734 5c0fd878779c
child 27739 d6d3cf5fda6f
permissions -rw-r--r--
changegroup: remove now-unused 'wasempty' variable and parameter
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
8226
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     1
# changegroup.py - Mercurial changegroup manipulation functions
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     2
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     3
#  Copyright 2006 Matt Mackall <mpm@selenic.com>
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     4
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     5
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9437
diff changeset
     6
# GNU General Public License version 2 or any later version.
3877
abaee83ce0a6 Replace demandload with new demandimport
Matt Mackall <mpm@selenic.com>
parents: 3859
diff changeset
     7
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
     8
from __future__ import absolute_import
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
     9
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    10
import os
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    11
import struct
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    12
import tempfile
20933
d3775db748a0 localrepo: move the addchangegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20932
diff changeset
    13
import weakref
25921
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    14
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    15
from .i18n import _
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    16
from .node import (
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    17
    hex,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    18
    nullid,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    19
    nullrev,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    20
    short,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    21
)
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    22
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    23
from . import (
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    24
    branchmap,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    25
    dagutil,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    26
    discovery,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    27
    error,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    28
    mdiff,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    29
    phases,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    30
    util,
74b303a637bc changegroup: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25831
diff changeset
    31
)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    32
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
    33
_CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
    34
_CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
    35
_CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH"
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
    36
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    37
def readexactly(stream, n):
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    38
    '''read n bytes from stream.read and abort if less was available'''
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    39
    s = stream.read(n)
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    40
    if len(s) < n:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
    41
        raise error.Abort(_("stream ended unexpectedly"
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    42
                           " (got %d bytes, expected %d)")
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    43
                          % (len(s), n))
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    44
    return s
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    45
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    46
def getchunk(stream):
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    47
    """return the next chunk from stream as a string"""
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    48
    d = readexactly(stream, 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    49
    l = struct.unpack(">l", d)[0]
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    50
    if l <= 4:
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
    51
        if l:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
    52
            raise error.Abort(_("invalid chunk length %d") % l)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    53
        return ""
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
    54
    return readexactly(stream, l - 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    55
5368
61462e7d62ed changegroup: avoid large copies
Matt Mackall <mpm@selenic.com>
parents: 3932
diff changeset
    56
def chunkheader(length):
9437
1c4e4004f3a6 Improve some docstrings relating to changegroups and prepush().
Greg Ward <greg-hg@gerg.ca>
parents: 9087
diff changeset
    57
    """return a changegroup chunk header (string)"""
5368
61462e7d62ed changegroup: avoid large copies
Matt Mackall <mpm@selenic.com>
parents: 3932
diff changeset
    58
    return struct.pack(">l", length + 4)
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    59
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    60
def closechunk():
9437
1c4e4004f3a6 Improve some docstrings relating to changegroups and prepush().
Greg Ward <greg-hg@gerg.ca>
parents: 9087
diff changeset
    61
    """return a changegroup chunk header (string) for a zero-length chunk"""
1981
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    62
    return struct.pack(">l", 0)
736b6c96bbbc make incoming work via ssh (issue139); move chunk code into separate module.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
diff changeset
    63
23890
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    64
def combineresults(results):
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    65
    """logic to combine 0 or more addchangegroup results into one"""
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    66
    changedheads = 0
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    67
    result = 1
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    68
    for ret in results:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    69
        # If any changegroup result is 0, return 0
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    70
        if ret == 0:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    71
            result = 0
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    72
            break
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    73
        if ret < -1:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    74
            changedheads += ret + 1
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    75
        elif ret > 1:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    76
            changedheads += ret - 1
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    77
    if changedheads > 0:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    78
        result = 1 + changedheads
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    79
    elif changedheads < 0:
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    80
        result = -1 + changedheads
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    81
    return result
7817059917d0 pullbundle2: extract addchangegroup result combining into its own function
Eric Sumner <ericsumner@fb.com>
parents: 23748
diff changeset
    82
3662
f4dc02d7fb71 unduplicate bundle writing code from httprepo
Matt Mackall <mpm@selenic.com>
parents: 3660
diff changeset
    83
bundletypes = {
26272
59c410db8c68 readbundle: fix typo of None compression
Yuya Nishihara <yuya@tcha.org>
parents: 26271
diff changeset
    84
    "": ("", None),       # only when using unbundle on ssh and old http servers
14060
aaa9a5989405 bundle: more comments about the different header types, remove useless if
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 13831
diff changeset
    85
                          # since the unification ssh accepts a header but there
aaa9a5989405 bundle: more comments about the different header types, remove useless if
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 13831
diff changeset
    86
                          # is no capability signaling it.
24686
e0e28e910fa3 bundle2: rename format, parts and config to final names
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 24190
diff changeset
    87
    "HG20": (), # special-cased below
26271
a0eff7ebc2ae readbundle: map 'HG10UN' to None compression
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26270
diff changeset
    88
    "HG10UN": ("HG10UN", None),
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
    89
    "HG10BZ": ("HG10", 'BZ'),
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
    90
    "HG10GZ": ("HG10GZ", 'GZ'),
3662
f4dc02d7fb71 unduplicate bundle writing code from httprepo
Matt Mackall <mpm@selenic.com>
parents: 3660
diff changeset
    91
}
f4dc02d7fb71 unduplicate bundle writing code from httprepo
Matt Mackall <mpm@selenic.com>
parents: 3660
diff changeset
    92
9087
f48454a279b9 typos: "it's" -> "its"
Martin Geisler <mg@lazybytes.net>
parents: 8312
diff changeset
    93
# hgweb uses this list to communicate its preferred type
6152
c050548307a4 hgweb: use bundletypes from mercurial.changegroup
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents: 5906
diff changeset
    94
bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
c050548307a4 hgweb: use bundletypes from mercurial.changegroup
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents: 5906
diff changeset
    95
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    96
def writechunks(ui, chunks, filename, vfs=None):
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    97
    """Write chunks to a file and return its filename.
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
    98
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
    99
    The stream is assumed to be a bundle file.
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   100
    Existing files will not be overwritten.
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   101
    If no filename is specified, a temporary file is created.
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   102
    """
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   103
    fh = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   104
    cleanup = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   105
    try:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   106
        if filename:
20976
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   107
            if vfs:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   108
                fh = vfs.open(filename, "wb")
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   109
            else:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   110
                fh = open(filename, "wb")
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   111
        else:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   112
            fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   113
            fh = os.fdopen(fd, "wb")
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   114
        cleanup = filename
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   115
        for c in chunks:
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   116
            fh.write(c)
3659
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   117
        cleanup = None
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   118
        return filename
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   119
    finally:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   120
        if fh is not None:
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   121
            fh.close()
025f68f22ae2 move write_bundle to changegroup.py
Matt Mackall <mpm@selenic.com>
parents: 2470
diff changeset
   122
        if cleanup is not None:
20976
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   123
            if filename and vfs:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   124
                vfs.unlink(cleanup)
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   125
            else:
c20f4898631e changegroup: add "vfs" argument to "writebundle()" for relative access via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20966
diff changeset
   126
                os.unlink(cleanup)
3660
8500a13ec44b create a readbundle function
Matt Mackall <mpm@selenic.com>
parents: 3659
diff changeset
   127
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   128
def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None):
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   129
    """Write a bundle file and return its filename.
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   130
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   131
    Existing files will not be overwritten.
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   132
    If no filename is specified, a temporary file is created.
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   133
    bz2 compression can be turned off.
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   134
    The bundle file will be deleted in case of errors.
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   135
    """
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   136
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   137
    if bundletype == "HG20":
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   138
        from . import bundle2
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   139
        bundle = bundle2.bundle20(ui)
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   140
        bundle.setcompression(compression)
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   141
        part = bundle.newpart('changegroup', data=cg.getchunks())
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   142
        part.addparam('version', cg.version)
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   143
        chunkiter = bundle.getchunks()
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   144
    else:
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   145
        # compression argument is only for the bundle2 case
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   146
        assert compression is None
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   147
        if cg.version != '01':
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   148
            raise error.Abort(_('old bundle types only supports v1 '
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   149
                                'changegroups'))
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   150
        header, comp = bundletypes[bundletype]
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   151
        if comp not in util.compressors:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   152
            raise error.Abort(_('unknown stream compression type: %s')
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   153
                              % comp)
26540
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   154
        z = util.compressors[comp]()
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   155
        subchunkiter = cg.getchunks()
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   156
        def chunkiter():
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   157
            yield header
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   158
            for chunk in subchunkiter:
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   159
                yield z.compress(chunk)
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   160
            yield z.flush()
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   161
        chunkiter = chunkiter()
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   162
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   163
    # parse the changegroup data, otherwise we will block
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   164
    # in case of sshrepo because we don't know the end of the stream
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   165
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   166
    # an empty chunkgroup is the end of the changegroup
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   167
    # a changegroup has at least 2 chunkgroups (changelog and manifest).
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   168
    # after that, an empty chunkgroup is the end of the changegroup
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   169
    return writechunks(ui, chunkiter, filename, vfs=vfs)
7469067de2ba changegroup: extract the file management part in its own function
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26509
diff changeset
   170
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   171
class cg1unpacker(object):
26708
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   172
    """Unpacker for cg1 changegroup streams.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   173
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   174
    A changegroup unpacker handles the framing of the revision data in
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   175
    the wire format. Most consumers will want to use the apply()
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   176
    method to add the changes from the changegroup to a repository.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   177
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   178
    If you're forwarding a changegroup unmodified to another consumer,
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   179
    use getchunks(), which returns an iterator of changegroup
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   180
    chunks. This is mostly useful for cases where you need to know the
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   181
    data stream has ended by observing the end of the changegroup.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   182
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   183
    deltachunk() is useful only if you're applying delta data. Most
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   184
    consumers should prefer apply() instead.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   185
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   186
    A few other public methods exist. Those are used only for
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   187
    bundlerepo and some debug commands - their use is discouraged.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   188
    """
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   189
    deltaheader = _CHANGEGROUPV1_DELTA_HEADER
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   190
    deltaheadersize = struct.calcsize(deltaheader)
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   191
    version = '01'
12043
bef5effb3db0 bundle: introduce bundle class
Matt Mackall <mpm@selenic.com>
parents: 12042
diff changeset
   192
    def __init__(self, fh, alg):
26267
eca468b8fae4 compression: use 'None' for no-compression
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26266
diff changeset
   193
        if alg == 'UN':
eca468b8fae4 compression: use 'None' for no-compression
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26266
diff changeset
   194
            alg = None # get more modern without breaking too much
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
   195
        if not alg in util.decompressors:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   196
            raise error.Abort(_('unknown stream compression type: %s')
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
   197
                             % alg)
26392
127b59787fd5 changegroup: use a different compression key for BZ in HG10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26272
diff changeset
   198
        if alg == 'BZ':
127b59787fd5 changegroup: use a different compression key for BZ in HG10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26272
diff changeset
   199
            alg = '_truncatedBZ'
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25921
diff changeset
   200
        self._stream = util.decompressors[alg](fh)
12044
bcc7139521b7 bundlerepo: remove duplication of bundle decompressors
Matt Mackall <mpm@selenic.com>
parents: 12043
diff changeset
   201
        self._type = alg
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   202
        self.callback = None
26706
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   203
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   204
    # These methods (compressed, read, seek, tell) all appear to only
8c0c3059f478 changegroup: note why a few methods on cg1unpacker exist
Augie Fackler <augie@google.com>
parents: 26704
diff changeset
   205
    # be used by bundlerepo, but it's a little hard to tell.
12044
bcc7139521b7 bundlerepo: remove duplication of bundle decompressors
Matt Mackall <mpm@selenic.com>
parents: 12043
diff changeset
   206
    def compressed(self):
26267
eca468b8fae4 compression: use 'None' for no-compression
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26266
diff changeset
   207
        return self._type is not None
12043
bef5effb3db0 bundle: introduce bundle class
Matt Mackall <mpm@selenic.com>
parents: 12042
diff changeset
   208
    def read(self, l):
bef5effb3db0 bundle: introduce bundle class
Matt Mackall <mpm@selenic.com>
parents: 12042
diff changeset
   209
        return self._stream.read(l)
12330
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   210
    def seek(self, pos):
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   211
        return self._stream.seek(pos)
e527b8635881 bundle: make unbundle object seekable
Matt Mackall <mpm@selenic.com>
parents: 12329
diff changeset
   212
    def tell(self):
12332
680fe77ab5b8 bundlerepo: use bundle objects everywhere
Matt Mackall <mpm@selenic.com>
parents: 12330
diff changeset
   213
        return self._stream.tell()
12347
6277a9469dff bundlerepo: restore close() method
Matt Mackall <mpm@selenic.com>
parents: 12336
diff changeset
   214
    def close(self):
6277a9469dff bundlerepo: restore close() method
Matt Mackall <mpm@selenic.com>
parents: 12336
diff changeset
   215
        return self._stream.close()
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   216
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   217
    def _chunklength(self):
13459
acbe171c8fbe changegroup: fix typo introduced in 9f2c407caf34
Jim Hague <jim.hague@acm.org>
parents: 13458
diff changeset
   218
        d = readexactly(self._stream, 4)
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   219
        l = struct.unpack(">l", d)[0]
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   220
        if l <= 4:
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   221
            if l:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   222
                raise error.Abort(_("invalid chunk length %d") % l)
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   223
            return 0
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   224
        if self.callback:
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   225
            self.callback()
13458
9f2c407caf34 changegroup: don't accept odd chunk headers
Mads Kiilerich <mads@kiilerich.com>
parents: 13457
diff changeset
   226
        return l - 4
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   227
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   228
    def changelogheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   229
        """v10 does not have a changelog header chunk"""
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   230
        return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   231
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   232
    def manifestheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   233
        """v10 does not have a manifest header chunk"""
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   234
        return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   235
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   236
    def filelogheader(self):
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   237
        """return the header of the filelogs chunk, v10 only has the filename"""
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   238
        l = self._chunklength()
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   239
        if not l:
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   240
            return {}
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   241
        fname = readexactly(self._stream, l)
20675
f8d50add83e1 changegroup: move from dict() construction to {} literals
Augie Fackler <raf@durin42.com>
parents: 19708
diff changeset
   242
        return {'filename': fname}
12334
50946802593d bundle: refactor progress callback
Matt Mackall <mpm@selenic.com>
parents: 12333
diff changeset
   243
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   244
    def _deltaheader(self, headertuple, prevnode):
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   245
        node, p1, p2, cs = headertuple
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   246
        if prevnode is None:
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   247
            deltabase = p1
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   248
        else:
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   249
            deltabase = prevnode
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   250
        flags = 0
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   251
        return node, p1, p2, deltabase, cs, flags
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   252
14144
3c3c53d8343a unbundler: separate delta and header parsing
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14143
diff changeset
   253
    def deltachunk(self, prevnode):
26707
5ee6bd529300 changegroup: mark cg1unpacker.chunklength as private
Augie Fackler <augie@google.com>
parents: 26706
diff changeset
   254
        l = self._chunklength()
12336
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   255
        if not l:
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   256
            return {}
14141
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   257
        headerdata = readexactly(self._stream, self.deltaheadersize)
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   258
        header = struct.unpack(self.deltaheader, headerdata)
bd1cbfe5db5c bundler: make parsechunk return the base revision of the delta
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14060
diff changeset
   259
        delta = readexactly(self._stream, l - self.deltaheadersize)
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   260
        node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
20675
f8d50add83e1 changegroup: move from dict() construction to {} literals
Augie Fackler <raf@durin42.com>
parents: 19708
diff changeset
   261
        return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   262
                'deltabase': deltabase, 'delta': delta, 'flags': flags}
12336
9d234f7d8a77 bundle: move chunk parsing into unbundle class
Matt Mackall <mpm@selenic.com>
parents: 12335
diff changeset
   263
20999
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   264
    def getchunks(self):
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   265
        """returns all the chunks contains in the bundle
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   266
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   267
        Used when you need to forward the binary stream to a file or another
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   268
        network API. To do so, it parse the changegroup data, otherwise it will
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   269
        block in case of sshrepo because it don't know the end of the stream.
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   270
        """
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   271
        # an empty chunkgroup is the end of the changegroup
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   272
        # a changegroup has at least 2 chunkgroups (changelog and manifest).
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   273
        # after that, an empty chunkgroup is the end of the changegroup
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   274
        empty = False
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   275
        count = 0
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   276
        while not empty or count <= 2:
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   277
            empty = True
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   278
            count += 1
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   279
            while True:
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   280
                chunk = getchunk(self)
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   281
                if not chunk:
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   282
                    break
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   283
                empty = False
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   284
                yield chunkheader(len(chunk))
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   285
                pos = 0
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   286
                while pos < len(chunk):
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   287
                    next = pos + 2**20
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   288
                    yield chunk[pos:next]
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   289
                    pos = next
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   290
            yield closechunk()
1e28ec9744bf changegroup: move chunk extraction into a getchunks method of unbundle10
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20978
diff changeset
   291
26712
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   292
    def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   293
        # We know that we'll never have more manifests than we had
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   294
        # changesets.
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   295
        self.callback = prog(_('manifests'), numchanges)
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   296
        # no need to check for empty manifest group here:
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   297
        # if the result of the merge of 1 and 2 is the same in 3 and 4,
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   298
        # no new manifest will be created and the manifest group will
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   299
        # be empty during the pull
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   300
        self.manifestheader()
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   301
        repo.manifest.addgroup(self, revmap, trp)
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   302
        repo.ui.progress(_('manifests'), None)
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   303
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   304
    def apply(self, repo, srctype, url, emptyok=False,
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   305
              targetphase=phases.draft, expectedtotal=None):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   306
        """Add the changegroup returned by source.read() to this repo.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   307
        srctype is a string like 'push', 'pull', or 'unbundle'.  url is
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   308
        the URL of the repo where this changegroup is coming from.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   309
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   310
        Return an integer summarizing the change to this repo:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   311
        - nothing changed or no source: 0
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   312
        - more heads than before: 1+added heads (2..n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   313
        - fewer heads than before: -1-removed heads (-2..-n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   314
        - number of heads stays the same: 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   315
        """
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   316
        repo = repo.unfiltered()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   317
        def csmap(x):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   318
            repo.ui.debug("add changeset %s\n" % short(x))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   319
            return len(cl)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   320
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   321
        def revmap(x):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   322
            return cl.rev(x)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   323
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   324
        changesets = files = revisions = 0
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   325
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   326
        tr = repo.transaction("\n".join([srctype, util.hidepassword(url)]))
26880
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   327
        try:
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   328
            # The transaction could have been created before and already
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   329
            # carries source information. In this case we use the top
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   330
            # level data. We overwrite the argument because we need to use
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   331
            # the top level value (if they exist) in this function.
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   332
            srctype = tr.hookargs.setdefault('source', srctype)
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   333
            url = tr.hookargs.setdefault('url', url)
26881
6b1ea6961554 changegroup: call 'prechangegroup' hook before setting up write delay
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26880
diff changeset
   334
            repo.hook('prechangegroup', throw=True, **tr.hookargs)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   335
26880
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   336
            # write changelog data to temp files so concurrent readers
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   337
            # will not see an inconsistent view
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   338
            cl = repo.changelog
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   339
            cl.delayupdate(tr)
fa7f8b686633 changegroup: fix the scope of a try finally
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26859
diff changeset
   340
            oldheads = cl.heads()
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   341
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   342
            trp = weakref.proxy(tr)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   343
            # pull off the changeset group
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   344
            repo.ui.status(_("adding changesets\n"))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   345
            clstart = len(cl)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   346
            class prog(object):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   347
                def __init__(self, step, total):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   348
                    self._step = step
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   349
                    self._total = total
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   350
                    self._count = 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   351
                def __call__(self):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   352
                    repo.ui.progress(self._step, self._count, unit=_('chunks'),
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   353
                                     total=self._total)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   354
                    self._count += 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   355
            self.callback = prog(_('changesets'), expectedtotal)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   356
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   357
            efiles = set()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   358
            def onchangelog(cl, node):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   359
                efiles.update(cl.read(node)[3])
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   360
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   361
            self.changelogheader()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   362
            srccontent = cl.addgroup(self, csmap, trp,
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   363
                                     addrevisioncb=onchangelog)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   364
            efiles = len(efiles)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   365
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   366
            if not (srccontent or emptyok):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   367
                raise error.Abort(_("received changelog group is empty"))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   368
            clend = len(cl)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   369
            changesets = clend - clstart
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   370
            repo.ui.progress(_('changesets'), None)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   371
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   372
            # pull off the manifest group
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   373
            repo.ui.status(_("adding manifests\n"))
26712
04176eaf911b changegroup: move manifest unpacking into its own method
Augie Fackler <augie@google.com>
parents: 26711
diff changeset
   374
            self._unpackmanifests(repo, revmap, trp, prog, changesets)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   375
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   376
            needfiles = {}
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   377
            if repo.ui.configbool('server', 'validate', default=False):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   378
                # validate incoming csets have their manifests
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   379
                for cset in xrange(clstart, clend):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   380
                    mfnode = repo.changelog.read(repo.changelog.node(cset))[0]
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   381
                    mfest = repo.manifest.readdelta(mfnode)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   382
                    # store file nodes we must see
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   383
                    for f, n in mfest.iteritems():
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   384
                        needfiles.setdefault(f, set()).add(n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   385
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   386
            # process the files
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   387
            repo.ui.status(_("adding file changes\n"))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   388
            self.callback = None
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   389
            pr = prog(_('files'), efiles)
26704
d7e614513413 changegroup: mark _addchangegroupfiles as module-private
Augie Fackler <augie@google.com>
parents: 26703
diff changeset
   390
            newrevs, newfiles = _addchangegroupfiles(
27735
bd37212c20ed changegroup: remove now-unused 'wasempty' variable and parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 27734
diff changeset
   391
                repo, self, revmap, trp, pr, needfiles)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   392
            revisions += newrevs
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   393
            files += newfiles
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   394
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   395
            dh = 0
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   396
            if oldheads:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   397
                heads = cl.heads()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   398
                dh = len(heads) - len(oldheads)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   399
                for h in heads:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   400
                    if h not in oldheads and repo[h].closesbranch():
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   401
                        dh -= 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   402
            htext = ""
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   403
            if dh:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   404
                htext = _(" (%+d heads)") % dh
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   405
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   406
            repo.ui.status(_("added %d changesets"
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   407
                             " with %d changes to %d files%s\n")
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   408
                             % (changesets, revisions, files, htext))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   409
            repo.invalidatevolatilesets()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   410
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   411
            if changesets > 0:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   412
                if 'node' not in tr.hookargs:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   413
                    tr.hookargs['node'] = hex(cl.node(clstart))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   414
                    hookargs = dict(tr.hookargs)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   415
                else:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   416
                    hookargs = dict(tr.hookargs)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   417
                    hookargs['node'] = hex(cl.node(clstart))
26751
520defbc0335 hook: centralize passing HG_PENDING to external hook process
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 26712
diff changeset
   418
                repo.hook('pretxnchangegroup', throw=True, **hookargs)
26695
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   419
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   420
            added = [cl.node(r) for r in xrange(clstart, clend)]
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   421
            publishing = repo.publishing()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   422
            if srctype in ('push', 'serve'):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   423
                # Old servers can not push the boundary themselves.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   424
                # New servers won't push the boundary if changeset already
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   425
                # exists locally as secret
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   426
                #
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   427
                # We should not use added here but the list of all change in
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   428
                # the bundle
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   429
                if publishing:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   430
                    phases.advanceboundary(repo, tr, phases.public, srccontent)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   431
                else:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   432
                    # Those changesets have been pushed from the outside, their
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   433
                    # phases are going to be pushed alongside. Therefor
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   434
                    # `targetphase` is ignored.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   435
                    phases.advanceboundary(repo, tr, phases.draft, srccontent)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   436
                    phases.retractboundary(repo, tr, phases.draft, added)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   437
            elif srctype != 'strip':
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   438
                # publishing only alter behavior during push
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   439
                #
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   440
                # strip should not touch boundary at all
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   441
                phases.retractboundary(repo, tr, targetphase, added)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   442
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   443
            if changesets > 0:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   444
                if srctype != 'strip':
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   445
                    # During strip, branchcache is invalid but coming call to
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   446
                    # `destroyed` will repair it.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   447
                    # In other case we can safely update cache on disk.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   448
                    branchmap.updatecache(repo.filtered('served'))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   449
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   450
                def runhooks():
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   451
                    # These hooks run when the lock releases, not when the
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   452
                    # transaction closes. So it's possible for the changelog
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   453
                    # to have changed since we last saw it.
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   454
                    if clstart >= len(repo):
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   455
                        return
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   456
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   457
                    # forcefully update the on-disk branch cache
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   458
                    repo.ui.debug("updating the branch cache\n")
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   459
                    repo.hook("changegroup", **hookargs)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   460
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   461
                    for n in added:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   462
                        args = hookargs.copy()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   463
                        args['node'] = hex(n)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   464
                        repo.hook("incoming", **args)
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   465
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   466
                    newheads = [h for h in repo.heads() if h not in oldheads]
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   467
                    repo.ui.log("incoming",
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   468
                                "%s incoming changes - new heads: %s\n",
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   469
                                len(added),
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   470
                                ', '.join([hex(c[:6]) for c in newheads]))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   471
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   472
                tr.addpostclose('changegroup-runhooks-%020i' % clstart,
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   473
                                lambda tr: repo._afterlock(runhooks))
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   474
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   475
            tr.close()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   476
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   477
        finally:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   478
            tr.release()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   479
            repo.ui.flush()
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   480
        # never return 0 here:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   481
        if dh < 0:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   482
            return dh - 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   483
        else:
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   484
            return dh + 1
1121fced5b20 changegroup: migrate addchangegroup() to forward to cg?unpacker.apply()
Augie Fackler <augie@google.com>
parents: 26694
diff changeset
   485
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   486
class cg2unpacker(cg1unpacker):
26708
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   487
    """Unpacker for cg2 streams.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   488
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   489
    cg2 streams add support for generaldelta, so the delta header
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   490
    format is slightly different. All other features about the data
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   491
    remain the same.
749d913f24b8 changegroup: document the public surface area of cg?unpackers
Augie Fackler <augie@google.com>
parents: 26707
diff changeset
   492
    """
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   493
    deltaheader = _CHANGEGROUPV2_DELTA_HEADER
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   494
    deltaheadersize = struct.calcsize(deltaheader)
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   495
    version = '02'
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   496
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   497
    def _deltaheader(self, headertuple, prevnode):
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   498
        node, p1, p2, deltabase, cs = headertuple
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   499
        flags = 0
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   500
        return node, p1, p2, deltabase, cs, flags
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   501
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   502
class cg3unpacker(cg2unpacker):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   503
    """Unpacker for cg3 streams.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   504
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   505
    cg3 streams add support for exchanging treemanifests and revlog
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   506
    flags, so the only changes from cg2 are the delta header and
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   507
    version number.
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   508
    """
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   509
    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   510
    deltaheadersize = struct.calcsize(deltaheader)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   511
    version = '03'
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   512
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   513
    def _deltaheader(self, headertuple, prevnode):
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   514
        node, p1, p2, deltabase, cs, flags = headertuple
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   515
        return node, p1, p2, deltabase, cs, flags
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   516
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   517
class headerlessfixup(object):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   518
    def __init__(self, fh, h):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   519
        self._h = h
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   520
        self._fh = fh
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   521
    def read(self, n):
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   522
        if self._h:
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   523
            d, self._h = self._h[:n], self._h[n:]
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   524
            if len(d) < n:
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
   525
                d += readexactly(self._fh, n - len(d))
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   526
            return d
13457
e74fe15dc7fd changegroup: verify all stream reads
Mads Kiilerich <mads@kiilerich.com>
parents: 13456
diff changeset
   527
        return readexactly(self._fh, n)
12329
7458de933f26 bundle: push chunkbuffer down into decompress
Matt Mackall <mpm@selenic.com>
parents: 12044
diff changeset
   528
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   529
def _moddirs(files):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   530
    """Given a set of modified files, find the list of modified directories.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   531
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   532
    This returns a list of (path to changed dir, changed dir) tuples,
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   533
    as that's what the one client needs anyway.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   534
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   535
    >>> _moddirs(['a/b/c.py', 'a/b/c.txt', 'a/d/e/f/g.txt', 'i.txt', ])
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   536
    [('/', 'a/'), ('a/', 'b/'), ('a/', 'd/'), ('a/d/', 'e/'), ('a/d/e/', 'f/')]
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   537
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   538
    """
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   539
    alldirs = set()
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   540
    for f in files:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   541
        path = f.split('/')[:-1]
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   542
        for i in xrange(len(path) - 1, -1, -1):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   543
            dn = '/'.join(path[:i])
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   544
            current = dn + '/', path[i] + '/'
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   545
            if current in alldirs:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   546
                break
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   547
            alldirs.add(current)
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   548
    return sorted(alldirs)
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   549
22390
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   550
class cg1packer(object):
e2806b8613ca changegroup: rename bundle-related functions and classes
Sune Foldager <cryo@cyanite.org>
parents: 22070
diff changeset
   551
    deltaheader = _CHANGEGROUPV1_DELTA_HEADER
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   552
    version = '01'
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   553
    def __init__(self, repo, bundlecaps=None):
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   554
        """Given a source repo, construct a bundler.
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   555
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   556
        bundlecaps is optional and can be used to specify the set of
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   557
        capabilities which can be used to build the bundle.
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   558
        """
19201
309c439cdbaa bundle-ng: add bundlecaps argument to getbundle() command
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19200
diff changeset
   559
        # Set of capabilities we can use to build the bundle.
309c439cdbaa bundle-ng: add bundlecaps argument to getbundle() command
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19200
diff changeset
   560
        if bundlecaps is None:
309c439cdbaa bundle-ng: add bundlecaps argument to getbundle() command
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19200
diff changeset
   561
            bundlecaps = set()
309c439cdbaa bundle-ng: add bundlecaps argument to getbundle() command
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19200
diff changeset
   562
        self._bundlecaps = bundlecaps
25831
578fc97904da generaldelta: mark experimental reordering option
Matt Mackall <mpm@selenic.com>
parents: 25823
diff changeset
   563
        # experimental config: bundle.reorder
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   564
        reorder = repo.ui.config('bundle', 'reorder', 'auto')
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   565
        if reorder == 'auto':
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   566
            reorder = None
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   567
        else:
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   568
            reorder = util.parsebool(reorder)
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   569
        self._repo = repo
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   570
        self._reorder = reorder
19208
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   571
        self._progress = repo.ui.progress
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   572
        if self._repo.ui.verbose and not self._repo.ui.debugflag:
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   573
            self._verbosenote = self._repo.ui.note
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   574
        else:
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   575
            self._verbosenote = lambda s: None
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   576
13831
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   577
    def close(self):
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   578
        return closechunk()
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   579
13831
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   580
    def fileheader(self, fname):
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   581
        return chunkheader(len(fname)) + fname
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   582
24912
e285b98c65cc changegroup.group: drop 'reorder' parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 24911
diff changeset
   583
    def group(self, nodelist, revlog, lookup, units=None):
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   584
        """Calculate a delta group, yielding a sequence of changegroup chunks
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   585
        (strings).
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   586
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   587
        Given a list of changeset revs, return a set of deltas and
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   588
        metadata corresponding to nodes. The first delta is
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   589
        first parent(nodelist[0]) -> nodelist[0], the receiver is
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   590
        guaranteed to have this parent as it has all history before
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   591
        these changesets. In the case firstparent is nullrev the
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   592
        changegroup starts with a full revision.
19208
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   593
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   594
        If units is not None, progress detail will be generated, units specifies
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   595
        the type of revlog that is touched (changelog, manifest, etc.).
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   596
        """
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   597
        # if we don't have any revisions touched by these changesets, bail
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   598
        if len(nodelist) == 0:
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   599
            yield self.close()
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   600
            return
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   601
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   602
        # for generaldelta revlogs, we linearize the revs; this will both be
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   603
        # much quicker and generate a much smaller bundle
24912
e285b98c65cc changegroup.group: drop 'reorder' parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 24911
diff changeset
   604
        if (revlog._generaldelta and self._reorder is None) or self._reorder:
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   605
            dag = dagutil.revlogdag(revlog)
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   606
            revs = set(revlog.rev(n) for n in nodelist)
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   607
            revs = dag.linearize(revs)
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   608
        else:
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   609
            revs = sorted([revlog.rev(n) for n in nodelist])
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   610
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   611
        # add the parent of the first rev
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   612
        p = revlog.parentrevs(revs[0])[0]
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   613
        revs.insert(0, p)
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   614
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   615
        # build deltas
19208
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   616
        total = len(revs) - 1
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   617
        msgbundling = _('bundling')
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   618
        for r in xrange(len(revs) - 1):
19208
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   619
            if units is not None:
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   620
                self._progress(msgbundling, r + 1, unit=units, total=total)
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   621
            prev, curr = revs[r], revs[r + 1]
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   622
            linknode = lookup(revlog.node(curr))
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   623
            for c in self.revchunk(revlog, curr, prev, linknode):
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   624
                yield c
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   625
24901
e9edd53770fb changegroup: close progress in same function as it's started
Martin von Zweigbergk <martinvonz@google.com>
parents: 24900
diff changeset
   626
        if units is not None:
e9edd53770fb changegroup: close progress in same function as it's started
Martin von Zweigbergk <martinvonz@google.com>
parents: 24900
diff changeset
   627
            self._progress(msgbundling, None)
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   628
        yield self.close()
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   629
19289
6ea1f858efd9 bundle: refactor changegroup prune to be its own function
Durham Goode <durham@fb.com>
parents: 19208
diff changeset
   630
    # filter any nodes that claim to be part of the known set
24896
9183cb6886ef changegroup: removed unused 'source' parameter from prune()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24717
diff changeset
   631
    def prune(self, revlog, missing, commonrevs):
19289
6ea1f858efd9 bundle: refactor changegroup prune to be its own function
Durham Goode <durham@fb.com>
parents: 19208
diff changeset
   632
        rr, rl = revlog.rev, revlog.linkrev
6ea1f858efd9 bundle: refactor changegroup prune to be its own function
Durham Goode <durham@fb.com>
parents: 19208
diff changeset
   633
        return [n for n in missing if rl(rr(n)) not in commonrevs]
6ea1f858efd9 bundle: refactor changegroup prune to be its own function
Durham Goode <durham@fb.com>
parents: 19208
diff changeset
   634
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   635
    def _packmanifests(self, mfnodes, tmfnodes, lookuplinknode):
26711
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   636
        """Pack flat manifests into a changegroup stream."""
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   637
        ml = self._repo.manifest
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   638
        size = 0
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   639
        for chunk in self.group(
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   640
                mfnodes, ml, lookuplinknode, units=_('manifests')):
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   641
            size += len(chunk)
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   642
            yield chunk
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   643
        self._verbosenote(_('%8.i (manifests)\n') % size)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   644
        # It looks odd to assert this here, but tmfnodes doesn't get
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   645
        # filled in until after we've called lookuplinknode for
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   646
        # sending root manifests, so the only way to tell the streams
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   647
        # got crossed is to check after we've done all the work.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   648
        assert not tmfnodes
26711
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   649
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   650
    def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   651
        '''yield a sequence of changegroup chunks (strings)'''
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   652
        repo = self._repo
24978
f52560c64953 changegroup: drop _changelog and _manifest properties
Martin von Zweigbergk <martinvonz@google.com>
parents: 24977
diff changeset
   653
        cl = repo.changelog
f52560c64953 changegroup: drop _changelog and _manifest properties
Martin von Zweigbergk <martinvonz@google.com>
parents: 24977
diff changeset
   654
        ml = repo.manifest
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   655
23381
cc0ff93d0c0c changegroup: fix file linkrevs during reorders (issue4462)
Durham Goode <durham@fb.com>
parents: 23226
diff changeset
   656
        clrevorder = {}
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   657
        mfs = {} # needed manifests
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   658
        tmfnodes = {}
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   659
        fnodes = {} # needed file nodes
27237
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   660
        # maps manifest node id -> set(changed files)
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   661
        mfchangedfiles = {}
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   662
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   663
        # Callback for the changelog, used to collect changed files and manifest
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   664
        # nodes.
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   665
        # Returns the linkrev node (identity in the changelog case).
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   666
        def lookupcl(x):
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   667
            c = cl.read(x)
23381
cc0ff93d0c0c changegroup: fix file linkrevs during reorders (issue4462)
Durham Goode <durham@fb.com>
parents: 23226
diff changeset
   668
            clrevorder[x] = len(clrevorder)
27237
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   669
            n = c[0]
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   670
            # record the first changeset introducing this manifest version
27237
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   671
            mfs.setdefault(n, x)
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   672
            # Record a complete list of potentially-changed files in
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   673
            # this manifest.
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   674
            mfchangedfiles.setdefault(n, set()).update(c[3])
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   675
            return x
19204
e9c5b1c246dc bundle-ng: move bundle generation to changegroup.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19202
diff changeset
   676
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   677
        self._verbosenote(_('uncompressed size of bundle content:\n'))
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   678
        size = 0
24912
e285b98c65cc changegroup.group: drop 'reorder' parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 24911
diff changeset
   679
        for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   680
            size += len(chunk)
23224
f4ab47ccefde changegroup: don't define lookupmf() until it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents: 22971
diff changeset
   681
            yield chunk
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   682
        self._verbosenote(_('%8.i (changelog)\n') % size)
23224
f4ab47ccefde changegroup: don't define lookupmf() until it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents: 22971
diff changeset
   683
24977
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   684
        # We need to make sure that the linkrev in the changegroup refers to
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   685
        # the first changeset that introduced the manifest or file revision.
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   686
        # The fastpath is usually safer than the slowpath, because the filelogs
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   687
        # are walked in revlog order.
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   688
        #
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   689
        # When taking the slowpath with reorder=None and the manifest revlog
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   690
        # uses generaldelta, the manifest may be walked in the "wrong" order.
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   691
        # Without 'clrevorder', we would get an incorrect linkrev (see fix in
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   692
        # cc0ff93d0c0c).
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   693
        #
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   694
        # When taking the fastpath, we are only vulnerable to reordering
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   695
        # of the changelog itself. The changelog never uses generaldelta, so
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   696
        # it is only reordered when reorder=True. To handle this case, we
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   697
        # simply take the slowpath, which already has the 'clrevorder' logic.
4289383cb9d2 changegroup: document the cases where reordering complicates linkrevs
Martin von Zweigbergk <martinvonz@google.com>
parents: 24976
diff changeset
   698
        # This was also fixed in cc0ff93d0c0c.
24976
147d8892fc4b changegroup: extract condition for linkrev fastpath
Martin von Zweigbergk <martinvonz@google.com>
parents: 24912
diff changeset
   699
        fastpathlinkrev = fastpathlinkrev and not self._reorder
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   700
        # Treemanifests don't work correctly with fastpathlinkrev
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   701
        # either, because we don't discover which directory nodes to
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   702
        # send along with files. This could probably be fixed.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   703
        fastpathlinkrev = fastpathlinkrev and (
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   704
            'treemanifest' not in repo.requirements)
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   705
        # Callback for the manifest, used to collect linkrevs for filelog
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   706
        # revisions.
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   707
        # Returns the linkrev node (collected in lookupcl).
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   708
        if fastpathlinkrev:
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   709
            lookupmflinknode = mfs.__getitem__
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   710
        else:
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   711
            def lookupmflinknode(x):
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   712
                """Callback for looking up the linknode for manifests.
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
   713
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   714
                Returns the linkrev node for the specified manifest.
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
   715
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   716
                SIDE EFFECT:
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   717
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   718
                1) fclnodes gets populated with the list of relevant
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   719
                   file nodes if we're not using fastpathlinkrev
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   720
                2) When treemanifests are in use, collects treemanifest nodes
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   721
                   to send
27219
beb60a898dd0 changegroup: document manifest linkrev callback some more
Augie Fackler <augie@google.com>
parents: 27218
diff changeset
   722
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   723
                Note that this means manifests must be completely sent to
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   724
                the client before you can trust the list of files and
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   725
                treemanifests to send.
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   726
                """
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   727
                clnode = mfs[x]
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   728
                # We no longer actually care about reading deltas of
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   729
                # the manifest here, because we already know the list
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   730
                # of changed files, so for treemanifests (which
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   731
                # lazily-load anyway to *generate* a readdelta) we can
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   732
                # just load them with read() and then we'll actually
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   733
                # be able to correctly load node IDs from the
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   734
                # submanifest entries.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   735
                if 'treemanifest' in repo.requirements:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   736
                    mdata = ml.read(x)
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   737
                else:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   738
                    mdata = ml.readfast(x)
27237
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   739
                for f in mfchangedfiles[x]:
27238
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   740
                    try:
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   741
                        n = mdata[f]
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   742
                    except KeyError:
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   743
                        continue
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   744
                    # record the first changeset introducing this filelog
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   745
                    # version
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   746
                    fclnodes = fnodes.setdefault(f, {})
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   747
                    fclnode = fclnodes.setdefault(n, clnode)
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   748
                    if clrevorder[clnode] < clrevorder[fclnode]:
c3dc03109401 changegroup: drop 'if True' that made the previous change clearer
Augie Fackler <augie@google.com>
parents: 27237
diff changeset
   749
                        fclnodes[n] = clnode
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   750
                # gather list of changed treemanifest nodes
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   751
                if 'treemanifest' in repo.requirements:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   752
                    submfs = {'/': mdata}
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   753
                    for dn, bn in _moddirs(mfchangedfiles[x]):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   754
                        submf = submfs[dn]
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   755
                        submf = submf._dirs[bn]
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   756
                        submfs[submf.dir()] = submf
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   757
                        tmfclnodes = tmfnodes.setdefault(submf.dir(), {})
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   758
                        tmfclnodes.setdefault(submf._node, clnode)
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   759
                        if clrevorder[clnode] < clrevorder[fclnode]:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   760
                            tmfclnodes[n] = clnode
27239
65c47779bcb5 changegroup: remove one special case from lookupmflinknode
Augie Fackler <augie@google.com>
parents: 27238
diff changeset
   761
                return clnode
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
   762
24899
cf1ea7566b20 changegroup: rename 'mf' to 'ml' to match 'cl', since it's a revlog
Martin von Zweigbergk <martinvonz@google.com>
parents: 24898
diff changeset
   763
        mfnodes = self.prune(ml, mfs, commonrevs)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   764
        for x in self._packmanifests(
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   765
            mfnodes, tmfnodes, lookupmflinknode):
26711
0ef0aec56090 changegroup: move manifest packing into a separate function
Augie Fackler <augie@google.com>
parents: 26710
diff changeset
   766
            yield x
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
   767
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
   768
        mfs.clear()
24898
0bbf061564cf changegroup: rename 'needed' to 'clrevs' to match 'clnodes'
Martin von Zweigbergk <martinvonz@google.com>
parents: 24897
diff changeset
   769
        clrevs = set(cl.rev(x) for x in clnodes)
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
   770
27240
94a3d6fdc315 changegroup: clean up file lookup function
Augie Fackler <augie@google.com>
parents: 27239
diff changeset
   771
        if not fastpathlinkrev:
94a3d6fdc315 changegroup: clean up file lookup function
Augie Fackler <augie@google.com>
parents: 27239
diff changeset
   772
            def linknodes(unused, fname):
94a3d6fdc315 changegroup: clean up file lookup function
Augie Fackler <augie@google.com>
parents: 27239
diff changeset
   773
                return fnodes.get(fname, {})
94a3d6fdc315 changegroup: clean up file lookup function
Augie Fackler <augie@google.com>
parents: 27239
diff changeset
   774
        else:
27241
ead8e7069998 changegroup: restate file linknode callback using generator expressions
Augie Fackler <augie@google.com>
parents: 27240
diff changeset
   775
            cln = cl.node
27240
94a3d6fdc315 changegroup: clean up file lookup function
Augie Fackler <augie@google.com>
parents: 27239
diff changeset
   776
            def linknodes(filerevlog, fname):
20936
bfb40168391c changegroup: remove unused variable caught by pyflakes
Sean Farley <sean.michael.farley@gmail.com>
parents: 19708
diff changeset
   777
                llr = filerevlog.linkrev
27241
ead8e7069998 changegroup: restate file linknode callback using generator expressions
Augie Fackler <augie@google.com>
parents: 27240
diff changeset
   778
                fln = filerevlog.node
ead8e7069998 changegroup: restate file linknode callback using generator expressions
Augie Fackler <augie@google.com>
parents: 27240
diff changeset
   779
                revs = ((r, llr(r)) for r in filerevlog)
ead8e7069998 changegroup: restate file linknode callback using generator expressions
Augie Fackler <augie@google.com>
parents: 27240
diff changeset
   780
                return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   781
27237
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   782
        changedfiles = set()
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   783
        for x in mfchangedfiles.itervalues():
c08814b48ae5 changegroup: avoid iterating the whole manifest
Augie Fackler <augie@google.com>
parents: 27219
diff changeset
   784
            changedfiles.update(x)
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   785
        for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   786
                                        source):
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   787
            yield chunk
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   788
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   789
        yield self.close()
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   790
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   791
        if clnodes:
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   792
            repo.hook('outgoing', node=hex(clnodes[0]), source=source)
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   793
24897
5c35a6040352 changegroup: document that 'source' parameter exists for extensions
Martin von Zweigbergk <martinvonz@google.com>
parents: 24896
diff changeset
   794
    # The 'source' parameter is useful for extensions
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   795
    def generatefiles(self, changedfiles, linknodes, commonrevs, source):
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   796
        repo = self._repo
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   797
        progress = self._progress
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   798
        msgbundling = _('bundling')
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   799
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   800
        total = len(changedfiles)
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   801
        # for progress output
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   802
        msgfiles = _('files')
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   803
        for i, fname in enumerate(sorted(changedfiles)):
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   804
            filerevlog = repo.file(fname)
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   805
            if not filerevlog:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
   806
                raise error.Abort(_("empty or missing revlog for %s") % fname)
19334
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   807
95a49112e7ab bundle: move file chunk generation to it's own function
Durham Goode <durham@fb.com>
parents: 19325
diff changeset
   808
            linkrevnodes = linknodes(filerevlog, fname)
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   809
            # Lookup for filenodes, we collected the linkrev nodes above in the
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   810
            # fastpath case and with lookupmf in the slowpath case.
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   811
            def lookupfilelog(x):
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   812
                return linkrevnodes[x]
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   813
24896
9183cb6886ef changegroup: removed unused 'source' parameter from prune()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24717
diff changeset
   814
            filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
19206
6308896b1d4a bundle-ng: simplify bundle10.generate
Sune Foldager <cryo@cyanite.org>
parents: 19204
diff changeset
   815
            if filenodes:
19208
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   816
                progress(msgbundling, i + 1, item=fname, unit=msgfiles,
0b564cf359a7 bundle-ng: move progress handling out of the linkrev callback
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19207
diff changeset
   817
                         total=total)
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   818
                h = self.fileheader(fname)
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   819
                size = len(h)
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   820
                yield h
24912
e285b98c65cc changegroup.group: drop 'reorder' parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 24911
diff changeset
   821
                for chunk in self.group(filenodes, filerevlog, lookupfilelog):
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   822
                    size += len(chunk)
19202
0455fc94ae00 bundle-ng: move gengroup into bundler, pass repo object to bundler
Sune Foldager <cryo@cyanite.org>
parents: 19201
diff changeset
   823
                    yield chunk
23748
4ab66de46a96 bundle: when verbose, show what takes up the space in the generated bundle
Mads Kiilerich <madski@unity3d.com>
parents: 23382
diff changeset
   824
                self._verbosenote(_('%8.i  %s\n') % (size, fname))
24901
e9edd53770fb changegroup: close progress in same function as it's started
Martin von Zweigbergk <martinvonz@google.com>
parents: 24900
diff changeset
   825
        progress(msgbundling, None)
19200
4cfdec944edf bundle-ng: move group into the bundler
Sune Foldager <cryo@cyanite.org>
parents: 19199
diff changeset
   826
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   827
    def deltaparent(self, revlog, rev, p1, p2, prev):
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   828
        return prev
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   829
19207
a67e1380dfbd bundle-ng: simplify lookup and state handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 19206
diff changeset
   830
    def revchunk(self, revlog, rev, prev, linknode):
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   831
        node = revlog.node(rev)
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   832
        p1, p2 = revlog.parentrevs(rev)
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   833
        base = self.deltaparent(revlog, rev, p1, p2, prev)
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   834
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   835
        prefix = ''
24190
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   836
        if revlog.iscensored(base) or revlog.iscensored(rev):
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   837
            try:
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   838
                delta = revlog.revision(node)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25624
diff changeset
   839
            except error.CensoredNodeError as e:
24190
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   840
                delta = e.tombstone
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   841
            if base == nullrev:
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   842
                prefix = mdiff.trivialdiffheader(len(delta))
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   843
            else:
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   844
                baselen = revlog.rawsize(base)
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   845
                prefix = mdiff.replacediffheader(baselen, len(delta))
903c7e8c97ad changegroup: emit full-replacement deltas if either revision is censored
Mike Edgar <adgar@google.com>
parents: 24180
diff changeset
   846
        elif base == nullrev:
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   847
            delta = revlog.revision(node)
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   848
            prefix = mdiff.trivialdiffheader(len(delta))
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   849
        else:
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   850
            delta = revlog.revdiff(base, rev)
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   851
        p1n, p2n = revlog.parents(node)
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   852
        basenode = revlog.node(base)
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   853
        flags = revlog.flags(rev)
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   854
        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   855
        meta += prefix
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   856
        l = len(meta) + len(delta)
13831
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   857
        yield chunkheader(l)
d69c9510d648 changegroup: introduce bundler objects
Matt Mackall <mpm@selenic.com>
parents: 13786
diff changeset
   858
        yield meta
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   859
        yield delta
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   860
    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   861
        # do nothing with basenode, it is implicitly the previous one in HG10
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   862
        # do nothing with flags, it is implicitly 0 for cg1 and cg2
14143
da635d3c5620 changegroup: new bundler API
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14141
diff changeset
   863
        return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
20925
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   864
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   865
class cg2packer(cg1packer):
23896
becfecaf9087 changegroup.writebundle: HG2Y support
Eric Sumner <ericsumner@fb.com>
parents: 23895
diff changeset
   866
    version = '02'
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   867
    deltaheader = _CHANGEGROUPV2_DELTA_HEADER
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   868
24911
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   869
    def __init__(self, repo, bundlecaps=None):
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   870
        super(cg2packer, self).__init__(repo, bundlecaps)
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   871
        if self._reorder is None:
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   872
            # Since generaldelta is directly supported by cg2, reordering
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   873
            # generally doesn't help, so we disable it by default (treating
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   874
            # bundle.reorder=auto just like bundle.reorder=False).
5447b8523fef cg2packer: set reorder=False in __init__ instead of in group()
Martin von Zweigbergk <martinvonz@google.com>
parents: 24910
diff changeset
   875
            self._reorder = False
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   876
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   877
    def deltaparent(self, revlog, rev, p1, p2, prev):
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   878
        dp = revlog.deltaparent(rev)
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   879
        # avoid storing full revisions; pick prev in those cases
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   880
        # also pick prev when we can't be sure remote has dp
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   881
        if dp == nullrev or (dp != p1 and dp != p2 and dp != prev):
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   882
            return prev
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   883
        return dp
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   884
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   885
    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   886
        # Do nothing with flags, it is implicitly 0 in cg1 and cg2
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   887
        return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   888
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   889
class cg3packer(cg2packer):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   890
    version = '03'
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   891
    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   892
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   893
    def _packmanifests(self, mfnodes, tmfnodes, lookuplinknode):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   894
        # Note that debug prints are super confusing in this code, as
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   895
        # tmfnodes gets populated by the calls to lookuplinknode in
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   896
        # the superclass's manifest packer. In the future we should
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   897
        # probably see if we can refactor this somehow to be less
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   898
        # confusing.
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   899
        for x in super(cg3packer, self)._packmanifests(
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   900
            mfnodes, {}, lookuplinknode):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   901
            yield x
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   902
        dirlog = self._repo.manifest.dirlog
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   903
        for name, nodes in tmfnodes.iteritems():
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   904
            # For now, directory headers are simply file headers with
27691
b0d23a55f91e changegroup: don't add a second trailing '/' in dir name
Martin von Zweigbergk <martinvonz@google.com>
parents: 27690
diff changeset
   905
            # a trailing '/' on the path (already in the name).
b0d23a55f91e changegroup: don't add a second trailing '/' in dir name
Martin von Zweigbergk <martinvonz@google.com>
parents: 27690
diff changeset
   906
            yield self.fileheader(name)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   907
            for chunk in self.group(nodes, dirlog(name), nodes.get):
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   908
                yield chunk
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   909
27433
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   910
    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   911
        return struct.pack(
12f727a5b434 changegroup: add flags field to cg3 delta header
Mike Edgar <adgar@google.com>
parents: 27432
diff changeset
   912
            self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   913
23181
832b7ef275c8 changegroup: introduce cg2packer/unpacker
Sune Foldager <cryo@cyanite.org>
parents: 23178
diff changeset
   914
packermap = {'01': (cg1packer, cg1unpacker),
26709
42733e956887 changegroup: reformat packermap and add comment
Augie Fackler <augie@google.com>
parents: 26708
diff changeset
   915
             # cg2 adds support for exchanging generaldelta
42733e956887 changegroup: reformat packermap and add comment
Augie Fackler <augie@google.com>
parents: 26708
diff changeset
   916
             '02': (cg2packer, cg2unpacker),
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   917
             # cg3 adds support for exchanging treemanifests
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
   918
             '03': (cg3packer, cg3unpacker),
26709
42733e956887 changegroup: reformat packermap and add comment
Augie Fackler <augie@google.com>
parents: 26708
diff changeset
   919
}
23168
a92ba36a1a9d changegroup: add a "packermap" dictionary to track different packer versions
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 22971
diff changeset
   920
20926
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   921
def _changegroupinfo(repo, nodes, source):
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   922
    if repo.ui.verbose or source == 'bundle':
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   923
        repo.ui.status(_("%d changesets found\n") % len(nodes))
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   924
    if repo.ui.debugflag:
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   925
        repo.ui.debug("list of changesets:\n")
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   926
        for node in nodes:
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   927
            repo.ui.debug("%s\n" % hex(node))
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   928
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   929
def getsubsetraw(repo, outgoing, bundler, source, fastpath=False):
20925
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   930
    repo = repo.unfiltered()
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   931
    commonrevs = outgoing.common
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   932
    csets = outgoing.missing
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   933
    heads = outgoing.missingheads
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   934
    # We go through the fast path if we get told to, or if all (unfiltered
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   935
    # heads have been requested (since we then know there all linkrevs will
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   936
    # be pulled by the client).
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   937
    heads.sort()
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   938
    fastpathlinkrev = fastpath or (
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   939
            repo.filtername is None and heads == sorted(repo.heads()))
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   940
5174c48ed8d8 localrepo: move the _changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20675
diff changeset
   941
    repo.hook('preoutgoing', throw=True, source=source)
20926
7c1ed40e3325 localrepo: move the changegroupinfo method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20925
diff changeset
   942
    _changegroupinfo(repo, csets, source)
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   943
    return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   944
26595
be0489770925 getsubset: get the unpacker version from the bundler
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26587
diff changeset
   945
def getsubset(repo, outgoing, bundler, source, fastpath=False):
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   946
    gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
26595
be0489770925 getsubset: get the unpacker version from the bundler
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26587
diff changeset
   947
    return packermap[bundler.version][1](util.chunkbuffer(gengroup), None)
20927
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   948
23897
f99a6e1865e5 changegroup.getsubset: support multiple versions
Eric Sumner <ericsumner@fb.com>
parents: 23896
diff changeset
   949
def changegroupsubset(repo, roots, heads, source, version='01'):
20927
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   950
    """Compute a changegroup consisting of all the nodes that are
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   951
    descendants of any of the roots and ancestors of any of the heads.
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   952
    Return a chunkbuffer object whose read() method will return
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   953
    successive changegroup chunks.
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   954
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   955
    It is fairly complex as determining which filenodes and which
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   956
    manifest nodes need to be included for the changeset to be complete
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   957
    is non-trivial.
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   958
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   959
    Another wrinkle is doing the reverse, figuring out which changeset in
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   960
    the changegroup a particular filenode or manifestnode belongs to.
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   961
    """
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   962
    cl = repo.changelog
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   963
    if not roots:
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   964
        roots = [nullid]
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   965
    discbases = []
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   966
    for n in roots:
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   967
        discbases.extend([p for p in cl.parents(n) if p != nullid])
25677
af5b2f4ed594 changegroup: properly compute common base in changeggroupsubset (issue4736)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 24717
diff changeset
   968
    # TODO: remove call to nodesbetween.
af5b2f4ed594 changegroup: properly compute common base in changeggroupsubset (issue4736)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 24717
diff changeset
   969
    csets, roots, heads = cl.nodesbetween(roots, heads)
af5b2f4ed594 changegroup: properly compute common base in changeggroupsubset (issue4736)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 24717
diff changeset
   970
    included = set(csets)
af5b2f4ed594 changegroup: properly compute common base in changeggroupsubset (issue4736)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 24717
diff changeset
   971
    discbases = [n for n in discbases if n not in included]
20927
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   972
    outgoing = discovery.outgoing(cl, discbases, heads)
23897
f99a6e1865e5 changegroup.getsubset: support multiple versions
Eric Sumner <ericsumner@fb.com>
parents: 23896
diff changeset
   973
    bundler = packermap[version][0](repo)
26595
be0489770925 getsubset: get the unpacker version from the bundler
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26587
diff changeset
   974
    return getsubset(repo, outgoing, bundler, source)
20927
24a443948627 localrepo: move the changegroupsubset method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20926
diff changeset
   975
23178
5e895ed5e955 changegroup: allow use of different cg#packer in getchangegroupraw
Sune Foldager <cryo@cyanite.org>
parents: 23177
diff changeset
   976
def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
5e895ed5e955 changegroup: allow use of different cg#packer in getchangegroupraw
Sune Foldager <cryo@cyanite.org>
parents: 23177
diff changeset
   977
                           version='01'):
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   978
    """Like getbundle, but taking a discovery.outgoing as an argument.
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   979
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   980
    This is only implemented for local repos and reuses potentially
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   981
    precomputed sets in outgoing. Returns a raw changegroup generator."""
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   982
    if not outgoing.missing:
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   983
        return None
23178
5e895ed5e955 changegroup: allow use of different cg#packer in getchangegroupraw
Sune Foldager <cryo@cyanite.org>
parents: 23177
diff changeset
   984
    bundler = packermap[version][0](repo, bundlecaps)
23177
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   985
    return getsubsetraw(repo, outgoing, bundler, source)
706547a14b8b changegroup: introduce "raw" versions of some commands
Sune Foldager <cryo@cyanite.org>
parents: 23168
diff changeset
   986
26508
47a12de9ac3f changegroup: add version argument to getlocalchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26424
diff changeset
   987
def getlocalchangegroup(repo, source, outgoing, bundlecaps=None,
47a12de9ac3f changegroup: add version argument to getlocalchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26424
diff changeset
   988
                        version='01'):
20928
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   989
    """Like getbundle, but taking a discovery.outgoing as an argument.
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   990
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   991
    This is only implemented for local repos and reuses potentially
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   992
    precomputed sets in outgoing."""
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   993
    if not outgoing.missing:
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   994
        return None
26508
47a12de9ac3f changegroup: add version argument to getlocalchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26424
diff changeset
   995
    bundler = packermap[version][0](repo, bundlecaps)
20928
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   996
    return getsubset(repo, outgoing, bundler, source)
91b47139d0cb localrepo: move the getlocalbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20927
diff changeset
   997
25400
7759dc97c5c7 changegroup: rename _computeoutgoing to computeoutgoing
Gregory Szorc <gregory.szorc@gmail.com>
parents: 24978
diff changeset
   998
def computeoutgoing(repo, heads, common):
21260
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
   999
    """Computes which revs are outgoing given a set of common
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1000
    and a set of heads.
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1001
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1002
    This is a separate function so extensions can have access to
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1003
    the logic.
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1004
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1005
    Returns a discovery.outgoing object.
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1006
    """
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1007
    cl = repo.changelog
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1008
    if common:
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1009
        hasnode = cl.hasnode
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1010
        common = [n for n in common if hasnode(n)]
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1011
    else:
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1012
        common = [nullid]
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1013
    if not heads:
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1014
        heads = cl.heads()
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1015
    return discovery.outgoing(cl, common, heads)
aa3e56607675 changegroup: refactor outgoing logic into a function
Durham Goode <durham@fb.com>
parents: 21153
diff changeset
  1016
26509
83d82fbefccb changegroup: add version argument to getchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26508
diff changeset
  1017
def getchangegroup(repo, source, heads=None, common=None, bundlecaps=None,
83d82fbefccb changegroup: add version argument to getchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26508
diff changeset
  1018
                   version='01'):
20930
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1019
    """Like changegroupsubset, but returns the set difference between the
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1020
    ancestors of heads and the ancestors common.
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1021
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1022
    If heads is None, use the local heads. If common is None, use [nullid].
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1023
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1024
    The nodes in common might not all be known locally due to the way the
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1025
    current discovery protocol works.
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1026
    """
25400
7759dc97c5c7 changegroup: rename _computeoutgoing to computeoutgoing
Gregory Szorc <gregory.szorc@gmail.com>
parents: 24978
diff changeset
  1027
    outgoing = computeoutgoing(repo, heads, common)
26509
83d82fbefccb changegroup: add version argument to getchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26508
diff changeset
  1028
    return getlocalchangegroup(repo, source, outgoing, bundlecaps=bundlecaps,
83d82fbefccb changegroup: add version argument to getchangegroup
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26508
diff changeset
  1029
                               version=version)
20930
4a987060d97e localrepo: move the getbundle method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20928
diff changeset
  1030
20931
de60ca3a390e localrepo: move the changegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20930
diff changeset
  1031
def changegroup(repo, basenodes, source):
de60ca3a390e localrepo: move the changegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20930
diff changeset
  1032
    # to avoid a race we use changegroupsubset() (issue1320)
de60ca3a390e localrepo: move the changegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20930
diff changeset
  1033
    return changegroupsubset(repo, basenodes, repo.heads(), source)
de60ca3a390e localrepo: move the changegroup method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20930
diff changeset
  1034
27735
bd37212c20ed changegroup: remove now-unused 'wasempty' variable and parameter
Martin von Zweigbergk <martinvonz@google.com>
parents: 27734
diff changeset
  1035
def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1036
    revisions = 0
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1037
    files = 0
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1038
    while True:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1039
        chunkdata = source.filelogheader()
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1040
        if not chunkdata:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1041
            break
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1042
        f = chunkdata["filename"]
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1043
        repo.ui.debug("adding %s revisions\n" % f)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1044
        pr()
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1045
        directory = (f[-1] == '/')
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1046
        if directory:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1047
            # a directory using treemanifests
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1048
            fl = repo.manifest.dirlog(f)
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1049
        else:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1050
            fl = repo.file(f)
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1051
        o = len(fl)
24120
a450e0a2ba0a revlog: in addgroup, reject ill-formed deltas based on censored nodes
Mike Edgar <adgar@google.com>
parents: 23897
diff changeset
  1052
        try:
a450e0a2ba0a revlog: in addgroup, reject ill-formed deltas based on censored nodes
Mike Edgar <adgar@google.com>
parents: 23897
diff changeset
  1053
            if not fl.addgroup(source, revmap, trp):
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1054
                raise error.Abort(_("received file revlog group is empty"))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25624
diff changeset
  1055
        except error.CensoredBaseError as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1056
            raise error.Abort(_("received delta base is censored: %s") % e)
27432
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1057
        if not directory:
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1058
            revisions += len(fl) - o
77d25b913f80 changegroup: introduce cg3, which has support for exchanging treemanifests
Augie Fackler <augie@google.com>
parents: 27241
diff changeset
  1059
            files += 1
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1060
        if f in needfiles:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1061
            needs = needfiles[f]
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1062
            for new in xrange(o, len(fl)):
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1063
                n = fl.node(new)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1064
                if n in needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1065
                    needs.remove(n)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1066
                else:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1067
                    raise error.Abort(
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1068
                        _("received spurious file revlog entry"))
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1069
            if not needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1070
                del needfiles[f]
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1071
    repo.ui.progress(_('files'), None)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1072
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1073
    for f, needs in needfiles.iteritems():
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1074
        fl = repo.file(f)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1075
        for n in needs:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1076
            try:
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1077
                fl.rev(n)
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1078
            except error.LookupError:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26540
diff changeset
  1079
                raise error.Abort(
20932
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1080
                    _('missing file data for %s:%s - run hg verify') %
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1081
                    (f, hex(n)))
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1082
0ac83e4e4f7c localrepo: move the addchangegroupfiles method in changegroup module
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20931
diff changeset
  1083
    return revisions, files