mercurial/streamclone.py
author Pierre-Yves David <pierre-yves.david@octobus.net>
Tue, 20 Apr 2021 04:27:03 +0200
changeset 46990 0b569c75d180
parent 46989 aed6ceaad6d7
child 47444 2f4ca4807033
child 47502 65c519661991
permissions -rw-r--r--
store: exclude `undo.` nodemap's file from `walk` There are "temporary" local file that we should not be transfered by `walk` user like local clone and stream clone. This fix the small issue that the new tests highlighted. Differential Revision: https://phab.mercurial-scm.org/D10482
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     1
# streamclone.py - producing and consuming streaming repository data
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     2
#
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     3
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     4
#
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     7
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     8
from __future__ import absolute_import
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     9
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
    10
import contextlib
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
    11
import os
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
    12
import struct
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
    13
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    14
from .i18n import _
43085
eef9a2d67051 py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
    15
from .pycompat import open
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    16
from .interfaces import repository
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    17
from . import (
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
    18
    cacheutil,
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    19
    error,
40339
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
    20
    narrowspec,
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
    21
    phases,
38164
aac4be30e250 py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents: 37638
diff changeset
    22
    pycompat,
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 45106
diff changeset
    23
    requirements as requirementsmod,
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
    24
    scmutil,
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
    25
    store,
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    26
    util,
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    27
)
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    28
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    29
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    30
def canperformstreamclone(pullop, bundle2=False):
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    31
    """Whether it is possible to perform a streaming clone as part of pull.
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
    32
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    33
    ``bundle2`` will cause the function to consider stream clone through
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    34
    bundle2 and only through bundle2.
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    35
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    36
    Returns a tuple of (supported, requirements). ``supported`` is True if
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    37
    streaming clone is supported and False otherwise. ``requirements`` is
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    38
    a set of repo requirements from the remote, or ``None`` if stream clone
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    39
    isn't supported.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    40
    """
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    41
    repo = pullop.repo
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    42
    remote = pullop.remote
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    43
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    44
    bundle2supported = False
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    45
    if pullop.canusebundle2:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    46
        if b'v2' in pullop.remotebundle2caps.get(b'stream', []):
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    47
            bundle2supported = True
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    48
        # else
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    49
        # Server doesn't support bundle2 stream clone or doesn't support
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    50
        # the versions we support. Fall back and possibly allow legacy.
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    51
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    52
    # Ensures legacy code path uses available bundle2.
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    53
    if bundle2supported and not bundle2:
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    54
        return False, None
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    55
    # Ensures bundle2 doesn't try to do a stream clone if it isn't supported.
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    56
    elif bundle2 and not bundle2supported:
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    57
        return False, None
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    58
26447
591088f7028a streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26446
diff changeset
    59
    # Streaming clone only works on empty repositories.
591088f7028a streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26446
diff changeset
    60
    if len(repo):
591088f7028a streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26446
diff changeset
    61
        return False, None
591088f7028a streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26446
diff changeset
    62
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    63
    # Streaming clone only works if all data is being requested.
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    64
    if pullop.heads:
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    65
        return False, None
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
    66
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    67
    streamrequested = pullop.streamclonerequested
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    68
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    69
    # If we don't have a preference, let the server decide for us. This
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    70
    # likely only comes into play in LANs.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    71
    if streamrequested is None:
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    72
        # The server can advertise whether to prefer streaming clone.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    73
        streamrequested = remote.capable(b'stream-preferred')
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    74
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    75
    if not streamrequested:
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    76
        return False, None
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
    77
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    78
    # In order for stream clone to work, the client has to support all the
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    79
    # requirements advertised by the server.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    80
    #
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    81
    # The server advertises its requirements via the "stream" and "streamreqs"
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    82
    # capability. "stream" (a value-less capability) is advertised if and only
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    83
    # if the only requirement is "revlogv1." Else, the "streamreqs" capability
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    84
    # is advertised and contains a comma-delimited list of requirements.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    85
    requirements = set()
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    86
    if remote.capable(b'stream'):
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 45106
diff changeset
    87
        requirements.add(requirementsmod.REVLOGV1_REQUIREMENT)
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    88
    else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    89
        streamreqs = remote.capable(b'streamreqs')
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    90
        # This is weird and shouldn't happen with modern servers.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    91
        if not streamreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    92
            pullop.repo.ui.warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    93
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    94
                    b'warning: stream clone requested but server has them '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    95
                    b'disabled\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    96
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    97
            )
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    98
            return False, None
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    99
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   100
        streamreqs = set(streamreqs.split(b','))
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   101
        # Server requires something we don't support. Bail.
32259
076f1ff43f0f clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents: 30975
diff changeset
   102
        missingreqs = streamreqs - repo.supportedformats
076f1ff43f0f clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents: 30975
diff changeset
   103
        if missingreqs:
076f1ff43f0f clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents: 30975
diff changeset
   104
            pullop.repo.ui.warn(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   105
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   106
                    b'warning: stream clone requested but client is missing '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   107
                    b'requirements: %s\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   108
                )
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   109
                % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   110
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   111
            pullop.repo.ui.warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   112
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   113
                    b'(see https://www.mercurial-scm.org/wiki/MissingRequirement '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   114
                    b'for more information)\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   115
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   116
            )
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   117
            return False, None
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   118
        requirements = streamreqs
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   119
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   120
    return True, requirements
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   121
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   122
26462
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   123
def maybeperformlegacystreamclone(pullop):
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   124
    """Possibly perform a legacy stream clone operation.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   125
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   126
    Legacy stream clones are performed as part of pull but before all other
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   127
    operations.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   128
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   129
    A legacy stream clone will not be performed if a bundle2 stream clone is
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   130
    supported.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   131
    """
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   132
    from . import localrepo
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   133
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   134
    supported, requirements = canperformstreamclone(pullop)
26458
362793295640 streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26447
diff changeset
   135
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   136
    if not supported:
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   137
        return
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   138
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   139
    repo = pullop.repo
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   140
    remote = pullop.remote
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   141
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   142
    # Save remote branchmap. We will use it later to speed up branchcache
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   143
    # creation.
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   144
    rbranchmap = None
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   145
    if remote.capable(b'branchmap'):
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   146
        with remote.commandexecutor() as e:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   147
            rbranchmap = e.callcommand(b'branchmap', {}).result()
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   148
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   149
    repo.ui.status(_(b'streaming all changes\n'))
26470
4b5647d9ee13 streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26469
diff changeset
   150
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   151
    with remote.commandexecutor() as e:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   152
        fp = e.callcommand(b'stream_out', {}).result()
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   153
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   154
    # TODO strictly speaking, this code should all be inside the context
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   155
    # manager because the context manager is supposed to ensure all wire state
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   156
    # is flushed when exiting. But the legacy peers don't do this, so it
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   157
    # doesn't matter.
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   158
    l = fp.readline()
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   159
    try:
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   160
        resp = int(l)
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   161
    except ValueError:
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   162
        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   163
            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   164
        )
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   165
    if resp == 1:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   166
        raise error.Abort(_(b'operation forbidden by server'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   167
    elif resp == 2:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   168
        raise error.Abort(_(b'locking the remote repository failed'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   169
    elif resp != 0:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   170
        raise error.Abort(_(b'the server sent an unknown error code'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   171
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   172
    l = fp.readline()
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   173
    try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   174
        filecount, bytecount = map(int, l.split(b' ', 1))
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   175
    except (ValueError, TypeError):
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   176
        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   177
            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   178
        )
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   179
27850
49cfddbf54ba with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents: 27845
diff changeset
   180
    with repo.lock():
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   181
        consumev1(repo, fp, filecount, bytecount)
26461
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   182
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   183
        # new requirements = old non-format requirements +
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   184
        #                    new format-related remote requirements
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   185
        # requirements from the streamed-in repository
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   186
        repo.requirements = requirements | (
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   187
            repo.requirements - repo.supportedformats
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   188
        )
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   189
        repo.svfs.options = localrepo.resolvestorevfsoptions(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   190
            repo.ui, repo.requirements, repo.features
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   191
        )
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
   192
        scmutil.writereporequirements(repo)
26461
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   193
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   194
        if rbranchmap:
41615
328ca3b9e545 branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents: 40494
diff changeset
   195
            repo._branchcaches.replace(repo, rbranchmap)
26461
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   196
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   197
        repo.invalidate()
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
   198
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   199
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   200
def allowservergeneration(repo):
26444
623743010133 streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26443
diff changeset
   201
    """Whether streaming clones are allowed from the server."""
40028
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   202
    if repository.REPO_FEATURE_STREAM_CLONE not in repo.features:
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   203
        return False
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   204
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   205
    if not repo.ui.configbool(b'server', b'uncompressed', untrusted=True):
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   206
        return False
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   207
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   208
    # The way stream clone works makes it impossible to hide secret changesets.
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   209
    # So don't allow this by default.
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   210
    secret = phases.hassecret(repo)
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   211
    if secret:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   212
        return repo.ui.configbool(b'server', b'uncompressedallowsecret')
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   213
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   214
    return True
26444
623743010133 streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26443
diff changeset
   215
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   216
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   217
# This is it's own function so extensions can override it.
40339
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   218
def _walkstreamfiles(repo, matcher=None):
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   219
    return repo.store.walk(matcher)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   220
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   221
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   222
def generatev1(repo):
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   223
    """Emit content for version 1 of a streaming clone.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   224
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   225
    This returns a 3-tuple of (file count, byte size, data iterator).
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   226
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   227
    The data iterator consists of N entries for each file being transferred.
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   228
    Each file entry starts as a line with the file name and integer size
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   229
    delimited by a null byte.
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   230
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   231
    The raw file data follows. Following the raw file data is the next file
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   232
    entry, or EOF.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   233
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   234
    When used on the wire protocol, an additional line indicating protocol
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   235
    success will be prepended to the stream. This function is not responsible
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   236
    for adding it.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   237
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   238
    This function will obtain a repository lock to ensure a consistent view of
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   239
    the store is captured. It therefore may raise LockError.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   240
    """
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   241
    entries = []
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   242
    total_bytes = 0
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   243
    # Get consistent snapshot of repo, lock during scan.
27845
7417e1c10253 with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents: 27794
diff changeset
   244
    with repo.lock():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   245
        repo.ui.debug(b'scanning\n')
46895
6085b7f1536d store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46626
diff changeset
   246
        for file_type, name, ename, size in _walkstreamfiles(repo):
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   247
            if size:
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   248
                entries.append((name, size))
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   249
                total_bytes += size
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   250
        _test_sync_point_walk_1(repo)
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   251
    _test_sync_point_walk_2(repo)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   252
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   253
    repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   254
        b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   255
    )
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   256
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   257
    svfs = repo.svfs
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   258
    debugflag = repo.ui.debugflag
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   259
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   260
    def emitrevlogdata():
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   261
        for name, size in entries:
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   262
            if debugflag:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   263
                repo.ui.debug(b'sending %s (%d bytes)\n' % (name, size))
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   264
            # partially encode name over the wire for backwards compat
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   265
            yield b'%s\0%d\n' % (store.encodedir(name), size)
33411
50b49bb0fff3 streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33410
diff changeset
   266
            # auditing at this stage is both pointless (paths are already
50b49bb0fff3 streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33410
diff changeset
   267
            # trusted by the local repo) and expensive
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   268
            with svfs(name, b'rb', auditpath=False) as fp:
33410
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   269
                if size <= 65536:
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   270
                    yield fp.read(size)
33410
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   271
                else:
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   272
                    for chunk in util.filechunkiter(fp, limit=size):
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   273
                        yield chunk
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   274
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   275
    return len(entries), total_bytes, emitrevlogdata()
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   276
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   277
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   278
def generatev1wireproto(repo):
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   279
    """Emit content for version 1 of streaming clone suitable for the wire.
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   280
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   281
    This is the data output from ``generatev1()`` with 2 header lines. The
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   282
    first line indicates overall success. The 2nd contains the file count and
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   283
    byte size of payload.
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   284
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   285
    The success line contains "0" for success, "1" for stream generation not
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   286
    allowed, and "2" for error locking the repository (possibly indicating
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   287
    a permissions error for the server process).
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   288
    """
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   289
    if not allowservergeneration(repo):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   290
        yield b'1\n'
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   291
        return
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   292
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   293
    try:
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   294
        filecount, bytecount, it = generatev1(repo)
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   295
    except error.LockError:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   296
        yield b'2\n'
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   297
        return
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   298
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   299
    # Indicates successful response.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   300
    yield b'0\n'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   301
    yield b'%d %d\n' % (filecount, bytecount)
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   302
    for chunk in it:
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   303
        yield chunk
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   304
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   305
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   306
def generatebundlev1(repo, compression=b'UN'):
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   307
    """Emit content for version 1 of a stream clone bundle.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   308
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   309
    The first 4 bytes of the output ("HGS1") denote this as stream clone
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   310
    bundle version 1.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   311
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   312
    The next 2 bytes indicate the compression type. Only "UN" is currently
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   313
    supported.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   314
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   315
    The next 16 bytes are two 64-bit big endian unsigned integers indicating
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   316
    file count and byte count, respectively.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   317
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   318
    The next 2 bytes is a 16-bit big endian unsigned short declaring the length
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   319
    of the requirements string, including a trailing \0. The following N bytes
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   320
    are the requirements string, which is ASCII containing a comma-delimited
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   321
    list of repo requirements that are needed to support the data.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   322
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   323
    The remaining content is the output of ``generatev1()`` (which may be
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   324
    compressed in the future).
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   325
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   326
    Returns a tuple of (requirements, data generator).
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   327
    """
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   328
    if compression != b'UN':
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   329
        raise ValueError(b'we do not support the compression argument yet')
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   330
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   331
    requirements = repo.requirements & repo.supportedformats
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   332
    requires = b','.join(sorted(requirements))
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   333
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   334
    def gen():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   335
        yield b'HGS1'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   336
        yield compression
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   337
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   338
        filecount, bytecount, it = generatev1(repo)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   339
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   340
            _(b'writing %d bytes for %d files\n') % (bytecount, filecount)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   341
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   342
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   343
        yield struct.pack(b'>QQ', filecount, bytecount)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   344
        yield struct.pack(b'>H', len(requires) + 1)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   345
        yield requires + b'\0'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   346
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   347
        # This is where we'll add compression in the future.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   348
        assert compression == b'UN'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   349
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   350
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   351
            _(b'bundle'), total=bytecount, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   352
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   353
        progress.update(0)
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   354
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   355
        for chunk in it:
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   356
            progress.increment(step=len(chunk))
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   357
            yield chunk
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   358
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   359
        progress.complete()
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   360
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   361
    return requirements, gen()
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   362
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   363
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   364
def consumev1(repo, fp, filecount, bytecount):
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   365
    """Apply the contents from version 1 of a streaming clone file handle.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   366
30332
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   367
    This takes the output from "stream_out" and applies it to the specified
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   368
    repository.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   369
30332
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   370
    Like "stream_out," the status line added by the wire protocol is not
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   371
    handled by this function.
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   372
    """
27859
f55a5ace8e69 with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents: 27850
diff changeset
   373
    with repo.lock():
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   374
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   375
            _(b'%d files to transfer, %s of data\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   376
            % (filecount, util.bytecount(bytecount))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   377
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   378
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   379
            _(b'clone'), total=bytecount, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   380
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   381
        progress.update(0)
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30332
diff changeset
   382
        start = util.timer()
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   383
29919
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   384
        # TODO: get rid of (potential) inconsistency
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   385
        #
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   386
        # If transaction is started and any @filecache property is
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   387
        # changed at this point, it causes inconsistency between
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   388
        # in-memory cached property and streamclone-ed file on the
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   389
        # disk. Nested transaction prevents transaction scope "clone"
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   390
        # below from writing in-memory changes out at the end of it,
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   391
        # even though in-memory changes are discarded at the end of it
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   392
        # regardless of transaction nesting.
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   393
        #
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   394
        # But transaction nesting can't be simply prohibited, because
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   395
        # nesting occurs also in ordinary case (e.g. enabling
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   396
        # clonebundles).
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   397
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   398
        with repo.transaction(b'clone'):
27897
2fdbf22a1b63 streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27896
diff changeset
   399
            with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount):
38783
e7aa113b14f7 global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38374
diff changeset
   400
                for i in pycompat.xrange(filecount):
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   401
                    # XXX doesn't support '\n' or '\r' in filenames
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   402
                    l = fp.readline()
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   403
                    try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   404
                        name, size = l.split(b'\0', 1)
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   405
                        size = int(size)
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   406
                    except (ValueError, TypeError):
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   407
                        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   408
                            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   409
                        )
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   410
                    if repo.ui.debugflag:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   411
                        repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   412
                            b'adding %s (%s)\n' % (name, util.bytecount(size))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   413
                        )
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   414
                    # for backwards compat, name was partially encoded
27897
2fdbf22a1b63 streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27896
diff changeset
   415
                    path = store.decodedir(name)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   416
                    with repo.svfs(path, b'w', backgroundclose=True) as ofp:
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   417
                        for chunk in util.filechunkiter(fp, limit=size):
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   418
                            progress.increment(step=len(chunk))
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   419
                            ofp.write(chunk)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   420
29919
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   421
            # force @filecache properties to be reloaded from
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   422
            # streamclone-ed file at next access
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   423
            repo.invalidate(clearfilecache=True)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   424
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30332
diff changeset
   425
        elapsed = util.timer() - start
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   426
        if elapsed <= 0:
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   427
            elapsed = 0.001
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   428
        progress.complete()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   429
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   430
            _(b'transferred %s in %.1f seconds (%s/sec)\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   431
            % (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   432
                util.bytecount(bytecount),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   433
                elapsed,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   434
                util.bytecount(bytecount / elapsed),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   435
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   436
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   437
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   438
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   439
def readbundle1header(fp):
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   440
    compression = fp.read(2)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   441
    if compression != b'UN':
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   442
        raise error.Abort(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   443
            _(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   444
                b'only uncompressed stream clone bundles are '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   445
                b'supported; got %s'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   446
            )
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   447
            % compression
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   448
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   449
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   450
    filecount, bytecount = struct.unpack(b'>QQ', fp.read(16))
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   451
    requireslen = struct.unpack(b'>H', fp.read(2))[0]
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   452
    requires = fp.read(requireslen)
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   453
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   454
    if not requires.endswith(b'\0'):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   455
        raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   456
            _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   457
                b'malformed stream clone bundle: '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   458
                b'requirements not properly encoded'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   459
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   460
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   461
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   462
    requirements = set(requires.rstrip(b'\0').split(b','))
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   463
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   464
    return filecount, bytecount, requirements
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   465
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   466
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   467
def applybundlev1(repo, fp):
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   468
    """Apply the content from a stream clone bundle version 1.
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   469
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   470
    We assume the 4 byte header has been read and validated and the file handle
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   471
    is at the 2 byte compression identifier.
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   472
    """
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   473
    if len(repo):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   474
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   475
            _(b'cannot apply stream clone bundle on non-empty repo')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   476
        )
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   477
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   478
    filecount, bytecount, requirements = readbundle1header(fp)
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   479
    missingreqs = requirements - repo.supportedformats
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   480
    if missingreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   481
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   482
            _(b'unable to apply stream clone: unsupported format: %s')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   483
            % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   484
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   485
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   486
    consumev1(repo, fp, filecount, bytecount)
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   487
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   488
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   489
class streamcloneapplier(object):
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   490
    """Class to manage applying streaming clone bundles.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   491
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   492
    We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   493
    readers to perform bundle type-specific functionality.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   494
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   495
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   496
    def __init__(self, fh):
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   497
        self._fh = fh
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   498
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   499
    def apply(self, repo):
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   500
        return applybundlev1(repo, self._fh)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   501
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   502
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   503
# type of file to stream
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   504
_fileappend = 0  # append only file
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   505
_filefull = 1  # full snapshot file
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   506
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   507
# Source of the file
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   508
_srcstore = b's'  # store (svfs)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   509
_srccache = b'c'  # cache (cache)
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   510
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   511
# This is it's own function so extensions can override it.
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   512
def _walkstreamfullstorefiles(repo):
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   513
    """list snapshot file from the store"""
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   514
    fnames = []
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   515
    if not repo.publishing():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   516
        fnames.append(b'phaseroots')
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   517
    return fnames
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   518
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   519
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   520
def _filterfull(entry, copy, vfsmap):
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   521
    """actually copy the snapshot files"""
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   522
    src, name, ftype, data = entry
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   523
    if ftype != _filefull:
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   524
        return entry
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   525
    return (src, name, ftype, copy(vfsmap[src].join(name)))
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   526
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   527
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   528
@contextlib.contextmanager
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   529
def maketempcopies():
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   530
    """return a function to temporary copy file"""
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   531
    files = []
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   532
    try:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   533
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   534
        def copy(src):
38164
aac4be30e250 py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents: 37638
diff changeset
   535
            fd, dst = pycompat.mkstemp()
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   536
            os.close(fd)
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   537
            files.append(dst)
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   538
            util.copyfiles(src, dst, hardlink=True)
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   539
            return dst
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   540
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   541
        yield copy
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   542
    finally:
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   543
        for tmp in files:
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   544
            util.tryunlink(tmp)
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   545
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   546
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   547
def _makemap(repo):
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   548
    """make a (src -> vfs) map for the repo"""
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   549
    vfsmap = {
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   550
        _srcstore: repo.svfs,
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   551
        _srccache: repo.cachevfs,
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   552
    }
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   553
    # we keep repo.vfs out of the on purpose, ther are too many danger there
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   554
    # (eg: .hg/hgrc)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   555
    assert repo.vfs not in vfsmap.values()
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   556
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   557
    return vfsmap
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   558
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   559
35802
bbc07357b567 streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents: 35767
diff changeset
   560
def _emit2(repo, entries, totalfilesize):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   561
    """actually emit the stream bundle"""
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   562
    vfsmap = _makemap(repo)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   563
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   564
        _(b'bundle'), total=totalfilesize, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   565
    )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   566
    progress.update(0)
38374
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   567
    with maketempcopies() as copy, progress:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   568
        # copy is delayed until we are in the try
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   569
        entries = [_filterfull(e, copy, vfsmap) for e in entries]
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   570
        yield None  # this release the lock on the repository
38374
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   571
        seen = 0
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   572
38374
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   573
        for src, name, ftype, data in entries:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   574
            vfs = vfsmap[src]
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   575
            yield src
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   576
            yield util.uvarintencode(len(name))
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   577
            if ftype == _fileappend:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   578
                fp = vfs(name)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   579
                size = data
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   580
            elif ftype == _filefull:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   581
                fp = open(data, b'rb')
38374
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   582
                size = util.fstat(fp).st_size
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   583
            try:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   584
                yield util.uvarintencode(size)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   585
                yield name
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   586
                if size <= 65536:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   587
                    chunks = (fp.read(size),)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   588
                else:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   589
                    chunks = util.filechunkiter(fp, limit=size)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   590
                for chunk in chunks:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   591
                    seen += len(chunk)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   592
                    progress.update(seen)
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   593
                    yield chunk
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   594
            finally:
800f5a2c869e progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents: 38373
diff changeset
   595
                fp.close()
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   596
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   597
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   598
def _test_sync_point_walk_1(repo):
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   599
    """a function for synchronisation during tests"""
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   600
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   601
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   602
def _test_sync_point_walk_2(repo):
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   603
    """a function for synchronisation during tests"""
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   604
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   605
40398
0ac794e0e285 streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents: 40339
diff changeset
   606
def generatev2(repo, includes, excludes, includeobsmarkers):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   607
    """Emit content for version 2 of a streaming clone.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   608
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   609
    the data stream consists the following entries:
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   610
    1) A char representing the file destination (eg: store or cache)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   611
    2) A varint containing the length of the filename
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   612
    3) A varint containing the length of file data
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   613
    4) N bytes containing the filename (the internal, store-agnostic form)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   614
    5) N bytes containing the file data
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   615
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   616
    Returns a 3-tuple of (file count, file size, data iterator).
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   617
    """
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   618
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   619
    with repo.lock():
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   620
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   621
        entries = []
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   622
        totalfilesize = 0
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   623
40339
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   624
        matcher = None
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   625
        if includes or excludes:
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   626
            matcher = narrowspec.match(repo.root, includes, excludes)
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   627
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   628
        repo.ui.debug(b'scanning\n')
46895
6085b7f1536d store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46626
diff changeset
   629
        for rl_type, name, ename, size in _walkstreamfiles(repo, matcher):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   630
            if size:
46989
aed6ceaad6d7 streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46986
diff changeset
   631
                ft = _fileappend
aed6ceaad6d7 streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46986
diff changeset
   632
                if rl_type & store.FILEFLAGS_VOLATILE:
aed6ceaad6d7 streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46986
diff changeset
   633
                    ft = _filefull
aed6ceaad6d7 streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46986
diff changeset
   634
                entries.append((_srcstore, name, ft, size))
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   635
                totalfilesize += size
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   636
        for name in _walkstreamfullstorefiles(repo):
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   637
            if repo.svfs.exists(name):
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   638
                totalfilesize += repo.svfs.lstat(name).st_size
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   639
                entries.append((_srcstore, name, _filefull, None))
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   640
        if includeobsmarkers and repo.svfs.exists(b'obsstore'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   641
            totalfilesize += repo.svfs.lstat(b'obsstore').st_size
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   642
            entries.append((_srcstore, b'obsstore', _filefull, None))
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   643
        for name in cacheutil.cachetocopy(repo):
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   644
            if repo.cachevfs.exists(name):
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   645
                totalfilesize += repo.cachevfs.lstat(name).st_size
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   646
                entries.append((_srccache, name, _filefull, None))
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   647
35802
bbc07357b567 streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents: 35767
diff changeset
   648
        chunks = _emit2(repo, entries, totalfilesize)
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   649
        first = next(chunks)
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   650
        assert first is None
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   651
        _test_sync_point_walk_1(repo)
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   652
    _test_sync_point_walk_2(repo)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   653
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   654
    return len(entries), totalfilesize, chunks
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   655
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   656
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   657
@contextlib.contextmanager
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   658
def nested(*ctxs):
39757
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   659
    this = ctxs[0]
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   660
    rest = ctxs[1:]
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   661
    with this:
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   662
        if rest:
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   663
            with nested(*rest):
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   664
                yield
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   665
        else:
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   666
            yield
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   667
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   668
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   669
def consumev2(repo, fp, filecount, filesize):
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   670
    """Apply the contents from a version 2 streaming clone.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   671
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   672
    Data is read from an object that only needs to provide a ``read(size)``
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   673
    method.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   674
    """
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   675
    with repo.lock():
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   676
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   677
            _(b'%d files to transfer, %s of data\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   678
            % (filecount, util.bytecount(filesize))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   679
        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   680
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   681
        start = util.timer()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   682
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   683
            _(b'clone'), total=filesize, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   684
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   685
        progress.update(0)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   686
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   687
        vfsmap = _makemap(repo)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   688
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   689
        with repo.transaction(b'clone'):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   690
            ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values())
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   691
            with nested(*ctxs):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   692
                for i in range(filecount):
35803
3ad3aaeb1134 streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents: 35802
diff changeset
   693
                    src = util.readexactly(fp, 1)
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   694
                    vfs = vfsmap[src]
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   695
                    namelen = util.uvarintdecodestream(fp)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   696
                    datalen = util.uvarintdecodestream(fp)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   697
35803
3ad3aaeb1134 streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents: 35802
diff changeset
   698
                    name = util.readexactly(fp, namelen)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   699
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   700
                    if repo.ui.debugflag:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   701
                        repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   702
                            b'adding [%s] %s (%s)\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   703
                            % (src, name, util.bytecount(datalen))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   704
                        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   705
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   706
                    with vfs(name, b'w') as ofp:
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   707
                        for chunk in util.filechunkiter(fp, limit=datalen):
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   708
                            progress.increment(step=len(chunk))
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   709
                            ofp.write(chunk)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   710
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   711
            # force @filecache properties to be reloaded from
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   712
            # streamclone-ed file at next access
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   713
            repo.invalidate(clearfilecache=True)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   714
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   715
        elapsed = util.timer() - start
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   716
        if elapsed <= 0:
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   717
            elapsed = 0.001
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   718
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   719
            _(b'transferred %s in %.1f seconds (%s/sec)\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   720
            % (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   721
                util.bytecount(progress.pos),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   722
                elapsed,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   723
                util.bytecount(progress.pos / elapsed),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   724
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   725
        )
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   726
        progress.complete()
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   727
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   728
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   729
def applybundlev2(repo, fp, filecount, filesize, requirements):
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   730
    from . import localrepo
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   731
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   732
    missingreqs = [r for r in requirements if r not in repo.supported]
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   733
    if missingreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   734
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   735
            _(b'unable to apply stream clone: unsupported format: %s')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   736
            % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   737
        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   738
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   739
    consumev2(repo, fp, filecount, filesize)
35804
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   740
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   741
    # new requirements = old non-format requirements +
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   742
    #                    new format-related remote requirements
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   743
    # requirements from the streamed-in repository
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   744
    repo.requirements = set(requirements) | (
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   745
        repo.requirements - repo.supportedformats
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   746
    )
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   747
    repo.svfs.options = localrepo.resolvestorevfsoptions(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   748
        repo.ui, repo.requirements, repo.features
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   749
    )
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
   750
    scmutil.writereporequirements(repo)