Mercurial > hg
annotate mercurial/streamclone.py @ 47532:ccbabaee5c36
dirstate-entry: add a `need_delay` method
This abstract the internal processing need for entry that would have an
ambiguous mtime (If I understand things correctly).
Differential Revision: https://phab.mercurial-scm.org/D10974
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Sun, 04 Jul 2021 02:13:53 +0200 |
parents | d370256636fe |
children | d7515d29761d |
rev | line source |
---|---|
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - producing and consuming streaming repository data |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
10 import contextlib |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
11 import errno |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
12 import os |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
13 import struct |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
14 |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
15 from .i18n import _ |
43085
eef9a2d67051
py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
16 from .pycompat import open |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
17 from .interfaces import repository |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 from . import ( |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
19 bookmarks, |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
20 cacheutil, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
21 error, |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
22 narrowspec, |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
23 phases, |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
24 pycompat, |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
25 requirements as requirementsmod, |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
26 scmutil, |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
27 store, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
28 util, |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 ) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
30 from .utils import ( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
31 stringutil, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
32 ) |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
34 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
35 def canperformstreamclone(pullop, bundle2=False): |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
36 """Whether it is possible to perform a streaming clone as part of pull. |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
37 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
38 ``bundle2`` will cause the function to consider stream clone through |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
39 bundle2 and only through bundle2. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
40 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
41 Returns a tuple of (supported, requirements). ``supported`` is True if |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
42 streaming clone is supported and False otherwise. ``requirements`` is |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
43 a set of repo requirements from the remote, or ``None`` if stream clone |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
44 isn't supported. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
45 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
46 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
47 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
48 |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
49 bundle2supported = False |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
50 if pullop.canusebundle2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
51 if b'v2' in pullop.remotebundle2caps.get(b'stream', []): |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
52 bundle2supported = True |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
53 # else |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
54 # Server doesn't support bundle2 stream clone or doesn't support |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
55 # the versions we support. Fall back and possibly allow legacy. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
56 |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
57 # Ensures legacy code path uses available bundle2. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
58 if bundle2supported and not bundle2: |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
59 return False, None |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
60 # Ensures bundle2 doesn't try to do a stream clone if it isn't supported. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
61 elif bundle2 and not bundle2supported: |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
62 return False, None |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
63 |
26447
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
64 # Streaming clone only works on empty repositories. |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
65 if len(repo): |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
66 return False, None |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
67 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
68 # Streaming clone only works if all data is being requested. |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
69 if pullop.heads: |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
70 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
71 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
72 streamrequested = pullop.streamclonerequested |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
73 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
74 # If we don't have a preference, let the server decide for us. This |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
75 # likely only comes into play in LANs. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
76 if streamrequested is None: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
77 # The server can advertise whether to prefer streaming clone. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
78 streamrequested = remote.capable(b'stream-preferred') |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
79 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
80 if not streamrequested: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
81 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
82 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
83 # In order for stream clone to work, the client has to support all the |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
84 # requirements advertised by the server. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
85 # |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
86 # The server advertises its requirements via the "stream" and "streamreqs" |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
87 # capability. "stream" (a value-less capability) is advertised if and only |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
88 # if the only requirement is "revlogv1." Else, the "streamreqs" capability |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
89 # is advertised and contains a comma-delimited list of requirements. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
90 requirements = set() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
91 if remote.capable(b'stream'): |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
92 requirements.add(requirementsmod.REVLOGV1_REQUIREMENT) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
93 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
94 streamreqs = remote.capable(b'streamreqs') |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
95 # This is weird and shouldn't happen with modern servers. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
96 if not streamreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
97 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
98 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
99 b'warning: stream clone requested but server has them ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
100 b'disabled\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
101 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
102 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
103 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
104 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
105 streamreqs = set(streamreqs.split(b',')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
106 # Server requires something we don't support. Bail. |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
107 missingreqs = streamreqs - repo.supportedformats |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
108 if missingreqs: |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
109 pullop.repo.ui.warn( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
110 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
111 b'warning: stream clone requested but client is missing ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
112 b'requirements: %s\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
113 ) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
114 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
115 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
116 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
117 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
118 b'(see https://www.mercurial-scm.org/wiki/MissingRequirement ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
119 b'for more information)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
120 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
121 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
122 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
123 requirements = streamreqs |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
124 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
125 return True, requirements |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
126 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
127 |
26462
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
128 def maybeperformlegacystreamclone(pullop): |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
129 """Possibly perform a legacy stream clone operation. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
130 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
131 Legacy stream clones are performed as part of pull but before all other |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
132 operations. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
133 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
134 A legacy stream clone will not be performed if a bundle2 stream clone is |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
135 supported. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
136 """ |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
137 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
138 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
139 supported, requirements = canperformstreamclone(pullop) |
26458
362793295640
streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26447
diff
changeset
|
140 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
141 if not supported: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
142 return |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
143 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
144 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
145 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
146 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
147 # Save remote branchmap. We will use it later to speed up branchcache |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
148 # creation. |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
149 rbranchmap = None |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
150 if remote.capable(b'branchmap'): |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
151 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
152 rbranchmap = e.callcommand(b'branchmap', {}).result() |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
153 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
154 repo.ui.status(_(b'streaming all changes\n')) |
26470
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
155 |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
156 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
157 fp = e.callcommand(b'stream_out', {}).result() |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
158 |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
159 # TODO strictly speaking, this code should all be inside the context |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
160 # manager because the context manager is supposed to ensure all wire state |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
161 # is flushed when exiting. But the legacy peers don't do this, so it |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
162 # doesn't matter. |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
163 l = fp.readline() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
164 try: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
165 resp = int(l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
166 except ValueError: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
167 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
168 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
169 ) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
170 if resp == 1: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
171 raise error.Abort(_(b'operation forbidden by server')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
172 elif resp == 2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
173 raise error.Abort(_(b'locking the remote repository failed')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
174 elif resp != 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
175 raise error.Abort(_(b'the server sent an unknown error code')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
176 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
177 l = fp.readline() |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
178 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
179 filecount, bytecount = map(int, l.split(b' ', 1)) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
180 except (ValueError, TypeError): |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
181 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
182 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
183 ) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
184 |
27850
49cfddbf54ba
with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents:
27845
diff
changeset
|
185 with repo.lock(): |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
186 consumev1(repo, fp, filecount, bytecount) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
187 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
188 # new requirements = old non-format requirements + |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
189 # new format-related remote requirements |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
190 # requirements from the streamed-in repository |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
191 repo.requirements = requirements | ( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
192 repo.requirements - repo.supportedformats |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
193 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
194 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
195 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
196 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
197 scmutil.writereporequirements(repo) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
198 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
199 if rbranchmap: |
41615
328ca3b9e545
branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents:
40494
diff
changeset
|
200 repo._branchcaches.replace(repo, rbranchmap) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
201 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
202 repo.invalidate() |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
203 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
204 |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
205 def allowservergeneration(repo): |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
206 """Whether streaming clones are allowed from the server.""" |
40028
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
207 if repository.REPO_FEATURE_STREAM_CLONE not in repo.features: |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
208 return False |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
209 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
210 if not repo.ui.configbool(b'server', b'uncompressed', untrusted=True): |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
211 return False |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
212 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
213 # The way stream clone works makes it impossible to hide secret changesets. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
214 # So don't allow this by default. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
215 secret = phases.hassecret(repo) |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
216 if secret: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
217 return repo.ui.configbool(b'server', b'uncompressedallowsecret') |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
218 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
219 return True |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
220 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
221 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
222 # This is it's own function so extensions can override it. |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
223 def _walkstreamfiles(repo, matcher=None): |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
224 return repo.store.walk(matcher) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
225 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
226 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
227 def generatev1(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
228 """Emit content for version 1 of a streaming clone. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
229 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
230 This returns a 3-tuple of (file count, byte size, data iterator). |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
231 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
232 The data iterator consists of N entries for each file being transferred. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
233 Each file entry starts as a line with the file name and integer size |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
234 delimited by a null byte. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
235 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
236 The raw file data follows. Following the raw file data is the next file |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
237 entry, or EOF. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
238 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
239 When used on the wire protocol, an additional line indicating protocol |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
240 success will be prepended to the stream. This function is not responsible |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
241 for adding it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
242 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
243 This function will obtain a repository lock to ensure a consistent view of |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
244 the store is captured. It therefore may raise LockError. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
245 """ |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
246 entries = [] |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
247 total_bytes = 0 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
248 # Get consistent snapshot of repo, lock during scan. |
27845
7417e1c10253
with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27794
diff
changeset
|
249 with repo.lock(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
250 repo.ui.debug(b'scanning\n') |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46626
diff
changeset
|
251 for file_type, name, ename, size in _walkstreamfiles(repo): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
252 if size: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
253 entries.append((name, size)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
254 total_bytes += size |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
255 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
256 _test_sync_point_walk_2(repo) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
257 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
258 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
259 b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
260 ) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
261 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
262 svfs = repo.svfs |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
263 debugflag = repo.ui.debugflag |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
264 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
265 def emitrevlogdata(): |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
266 for name, size in entries: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
267 if debugflag: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
268 repo.ui.debug(b'sending %s (%d bytes)\n' % (name, size)) |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
269 # partially encode name over the wire for backwards compat |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
270 yield b'%s\0%d\n' % (store.encodedir(name), size) |
33411
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
271 # auditing at this stage is both pointless (paths are already |
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
272 # trusted by the local repo) and expensive |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
273 with svfs(name, b'rb', auditpath=False) as fp: |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
274 if size <= 65536: |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
275 yield fp.read(size) |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
276 else: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
277 for chunk in util.filechunkiter(fp, limit=size): |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
278 yield chunk |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
279 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
280 return len(entries), total_bytes, emitrevlogdata() |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
281 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
282 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
283 def generatev1wireproto(repo): |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
284 """Emit content for version 1 of streaming clone suitable for the wire. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
285 |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
286 This is the data output from ``generatev1()`` with 2 header lines. The |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
287 first line indicates overall success. The 2nd contains the file count and |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
288 byte size of payload. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
289 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
290 The success line contains "0" for success, "1" for stream generation not |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
291 allowed, and "2" for error locking the repository (possibly indicating |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
292 a permissions error for the server process). |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
293 """ |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
294 if not allowservergeneration(repo): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
295 yield b'1\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
296 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
297 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
298 try: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
299 filecount, bytecount, it = generatev1(repo) |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
300 except error.LockError: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
301 yield b'2\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
302 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
303 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
304 # Indicates successful response. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
305 yield b'0\n' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
306 yield b'%d %d\n' % (filecount, bytecount) |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
307 for chunk in it: |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
308 yield chunk |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
309 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
310 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
311 def generatebundlev1(repo, compression=b'UN'): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
312 """Emit content for version 1 of a stream clone bundle. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
313 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
314 The first 4 bytes of the output ("HGS1") denote this as stream clone |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
315 bundle version 1. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
316 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
317 The next 2 bytes indicate the compression type. Only "UN" is currently |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
318 supported. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
319 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
320 The next 16 bytes are two 64-bit big endian unsigned integers indicating |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
321 file count and byte count, respectively. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
322 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
323 The next 2 bytes is a 16-bit big endian unsigned short declaring the length |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
324 of the requirements string, including a trailing \0. The following N bytes |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
325 are the requirements string, which is ASCII containing a comma-delimited |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
326 list of repo requirements that are needed to support the data. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
327 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
328 The remaining content is the output of ``generatev1()`` (which may be |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
329 compressed in the future). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
330 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
331 Returns a tuple of (requirements, data generator). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
332 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
333 if compression != b'UN': |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
334 raise ValueError(b'we do not support the compression argument yet') |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
335 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
336 requirements = repo.requirements & repo.supportedformats |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
337 requires = b','.join(sorted(requirements)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
338 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
339 def gen(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
340 yield b'HGS1' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
341 yield compression |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
342 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
343 filecount, bytecount, it = generatev1(repo) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
344 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
345 _(b'writing %d bytes for %d files\n') % (bytecount, filecount) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
346 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
347 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
348 yield struct.pack(b'>QQ', filecount, bytecount) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
349 yield struct.pack(b'>H', len(requires) + 1) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
350 yield requires + b'\0' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
351 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
352 # This is where we'll add compression in the future. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
353 assert compression == b'UN' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
354 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
355 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
356 _(b'bundle'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
357 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
358 progress.update(0) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
359 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
360 for chunk in it: |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
361 progress.increment(step=len(chunk)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
362 yield chunk |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
363 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
364 progress.complete() |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
365 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
366 return requirements, gen() |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
367 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
368 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
369 def consumev1(repo, fp, filecount, bytecount): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
370 """Apply the contents from version 1 of a streaming clone file handle. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
371 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
372 This takes the output from "stream_out" and applies it to the specified |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
373 repository. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
374 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
375 Like "stream_out," the status line added by the wire protocol is not |
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
376 handled by this function. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
377 """ |
27859
f55a5ace8e69
with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27850
diff
changeset
|
378 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
379 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
380 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
381 % (filecount, util.bytecount(bytecount)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
382 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
383 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
384 _(b'clone'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
385 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
386 progress.update(0) |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
387 start = util.timer() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
388 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
389 # TODO: get rid of (potential) inconsistency |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
390 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
391 # If transaction is started and any @filecache property is |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
392 # changed at this point, it causes inconsistency between |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
393 # in-memory cached property and streamclone-ed file on the |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
394 # disk. Nested transaction prevents transaction scope "clone" |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
395 # below from writing in-memory changes out at the end of it, |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
396 # even though in-memory changes are discarded at the end of it |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
397 # regardless of transaction nesting. |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
398 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
399 # But transaction nesting can't be simply prohibited, because |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
400 # nesting occurs also in ordinary case (e.g. enabling |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
401 # clonebundles). |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
402 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
403 with repo.transaction(b'clone'): |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
404 with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
38783
e7aa113b14f7
global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38374
diff
changeset
|
405 for i in pycompat.xrange(filecount): |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
406 # XXX doesn't support '\n' or '\r' in filenames |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
407 l = fp.readline() |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
408 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
409 name, size = l.split(b'\0', 1) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
410 size = int(size) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
411 except (ValueError, TypeError): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
412 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
413 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
414 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
415 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
416 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
417 b'adding %s (%s)\n' % (name, util.bytecount(size)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
418 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
419 # for backwards compat, name was partially encoded |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
420 path = store.decodedir(name) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
421 with repo.svfs(path, b'w', backgroundclose=True) as ofp: |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
422 for chunk in util.filechunkiter(fp, limit=size): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
423 progress.increment(step=len(chunk)) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
424 ofp.write(chunk) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
425 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
426 # force @filecache properties to be reloaded from |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
427 # streamclone-ed file at next access |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
428 repo.invalidate(clearfilecache=True) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
429 |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
430 elapsed = util.timer() - start |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
431 if elapsed <= 0: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
432 elapsed = 0.001 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
433 progress.complete() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
434 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
435 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
436 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
437 util.bytecount(bytecount), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
438 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
439 util.bytecount(bytecount / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
440 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
441 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
442 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
443 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
444 def readbundle1header(fp): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
445 compression = fp.read(2) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
446 if compression != b'UN': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
447 raise error.Abort( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
448 _( |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
449 b'only uncompressed stream clone bundles are ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
450 b'supported; got %s' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
451 ) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
452 % compression |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
453 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
454 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
455 filecount, bytecount = struct.unpack(b'>QQ', fp.read(16)) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
456 requireslen = struct.unpack(b'>H', fp.read(2))[0] |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
457 requires = fp.read(requireslen) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
458 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
459 if not requires.endswith(b'\0'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
460 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
461 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
462 b'malformed stream clone bundle: ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
463 b'requirements not properly encoded' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
464 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
465 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
466 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
467 requirements = set(requires.rstrip(b'\0').split(b',')) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
468 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
469 return filecount, bytecount, requirements |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
470 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
471 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
472 def applybundlev1(repo, fp): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
473 """Apply the content from a stream clone bundle version 1. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
474 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
475 We assume the 4 byte header has been read and validated and the file handle |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
476 is at the 2 byte compression identifier. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
477 """ |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
478 if len(repo): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
479 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
480 _(b'cannot apply stream clone bundle on non-empty repo') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
481 ) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
482 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
483 filecount, bytecount, requirements = readbundle1header(fp) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
484 missingreqs = requirements - repo.supportedformats |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
485 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
486 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
487 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
488 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
489 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
490 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
491 consumev1(repo, fp, filecount, bytecount) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
492 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
493 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
494 class streamcloneapplier(object): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
495 """Class to manage applying streaming clone bundles. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
496 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
497 We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
498 readers to perform bundle type-specific functionality. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
499 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
500 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
501 def __init__(self, fh): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
502 self._fh = fh |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
503 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
504 def apply(self, repo): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
505 return applybundlev1(repo, self._fh) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
506 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
507 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
508 # type of file to stream |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
509 _fileappend = 0 # append only file |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
510 _filefull = 1 # full snapshot file |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
511 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
512 # Source of the file |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
513 _srcstore = b's' # store (svfs) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
514 _srccache = b'c' # cache (cache) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
515 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
516 # This is it's own function so extensions can override it. |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
517 def _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
518 """list snapshot file from the store""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
519 fnames = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
520 if not repo.publishing(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
521 fnames.append(b'phaseroots') |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
522 return fnames |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
523 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
524 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
525 def _filterfull(entry, copy, vfsmap): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
526 """actually copy the snapshot files""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
527 src, name, ftype, data = entry |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
528 if ftype != _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
529 return entry |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
530 return (src, name, ftype, copy(vfsmap[src].join(name))) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
531 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
532 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
533 @contextlib.contextmanager |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
534 def maketempcopies(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
535 """return a function to temporary copy file""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
536 files = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
537 try: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
538 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
539 def copy(src): |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
540 fd, dst = pycompat.mkstemp() |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
541 os.close(fd) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
542 files.append(dst) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
543 util.copyfiles(src, dst, hardlink=True) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
544 return dst |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
545 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
546 yield copy |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
547 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
548 for tmp in files: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
549 util.tryunlink(tmp) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
550 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
551 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
552 def _makemap(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
553 """make a (src -> vfs) map for the repo""" |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
554 vfsmap = { |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
555 _srcstore: repo.svfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
556 _srccache: repo.cachevfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
557 } |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
558 # we keep repo.vfs out of the on purpose, ther are too many danger there |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
559 # (eg: .hg/hgrc) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
560 assert repo.vfs not in vfsmap.values() |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
561 |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
562 return vfsmap |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
563 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
564 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
565 def _emit2(repo, entries, totalfilesize): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
566 """actually emit the stream bundle""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
567 vfsmap = _makemap(repo) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
568 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
569 _(b'bundle'), total=totalfilesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
570 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
571 progress.update(0) |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
572 with maketempcopies() as copy, progress: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
573 # copy is delayed until we are in the try |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
574 entries = [_filterfull(e, copy, vfsmap) for e in entries] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
575 yield None # this release the lock on the repository |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
576 seen = 0 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
577 |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
578 for src, name, ftype, data in entries: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
579 vfs = vfsmap[src] |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
580 yield src |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
581 yield util.uvarintencode(len(name)) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
582 if ftype == _fileappend: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
583 fp = vfs(name) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
584 size = data |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
585 elif ftype == _filefull: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
586 fp = open(data, b'rb') |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
587 size = util.fstat(fp).st_size |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
588 try: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
589 yield util.uvarintencode(size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
590 yield name |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
591 if size <= 65536: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
592 chunks = (fp.read(size),) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
593 else: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
594 chunks = util.filechunkiter(fp, limit=size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
595 for chunk in chunks: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
596 seen += len(chunk) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
597 progress.update(seen) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
598 yield chunk |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
599 finally: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
600 fp.close() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
601 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
602 |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
603 def _test_sync_point_walk_1(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
604 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
605 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
606 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
607 def _test_sync_point_walk_2(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
608 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
609 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
610 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
611 def _v2_walk(repo, includes, excludes, includeobsmarkers): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
612 """emit a seris of files information useful to clone a repo |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
613 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
614 return (entries, totalfilesize) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
615 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
616 entries is a list of tuple (vfs-key, file-path, file-type, size) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
617 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
618 - `vfs-key`: is a key to the right vfs to write the file (see _makemap) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
619 - `name`: file path of the file to copy (to be feed to the vfss) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
620 - `file-type`: do this file need to be copied with the source lock ? |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
621 - `size`: the size of the file (or None) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
622 """ |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
623 assert repo._currentlock(repo._lockref) is not None |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
624 entries = [] |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
625 totalfilesize = 0 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
626 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
627 matcher = None |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
628 if includes or excludes: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
629 matcher = narrowspec.match(repo.root, includes, excludes) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
630 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
631 for rl_type, name, ename, size in _walkstreamfiles(repo, matcher): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
632 if size: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
633 ft = _fileappend |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
634 if rl_type & store.FILEFLAGS_VOLATILE: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
635 ft = _filefull |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
636 entries.append((_srcstore, name, ft, size)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
637 totalfilesize += size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
638 for name in _walkstreamfullstorefiles(repo): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
639 if repo.svfs.exists(name): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
640 totalfilesize += repo.svfs.lstat(name).st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
641 entries.append((_srcstore, name, _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
642 if includeobsmarkers and repo.svfs.exists(b'obsstore'): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
643 totalfilesize += repo.svfs.lstat(b'obsstore').st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
644 entries.append((_srcstore, b'obsstore', _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
645 for name in cacheutil.cachetocopy(repo): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
646 if repo.cachevfs.exists(name): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
647 totalfilesize += repo.cachevfs.lstat(name).st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
648 entries.append((_srccache, name, _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
649 return entries, totalfilesize |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
650 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
651 |
40398
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
652 def generatev2(repo, includes, excludes, includeobsmarkers): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
653 """Emit content for version 2 of a streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
654 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
655 the data stream consists the following entries: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
656 1) A char representing the file destination (eg: store or cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
657 2) A varint containing the length of the filename |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
658 3) A varint containing the length of file data |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
659 4) N bytes containing the filename (the internal, store-agnostic form) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
660 5) N bytes containing the file data |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
661 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
662 Returns a 3-tuple of (file count, file size, data iterator). |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
663 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
664 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
665 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
666 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
667 repo.ui.debug(b'scanning\n') |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
668 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
669 entries, totalfilesize = _v2_walk( |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
670 repo, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
671 includes=includes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
672 excludes=excludes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
673 includeobsmarkers=includeobsmarkers, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
674 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
675 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
676 chunks = _emit2(repo, entries, totalfilesize) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
677 first = next(chunks) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
678 assert first is None |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
679 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
680 _test_sync_point_walk_2(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
681 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
682 return len(entries), totalfilesize, chunks |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
683 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
684 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
685 @contextlib.contextmanager |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
686 def nested(*ctxs): |
39757
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
687 this = ctxs[0] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
688 rest = ctxs[1:] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
689 with this: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
690 if rest: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
691 with nested(*rest): |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
692 yield |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
693 else: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
694 yield |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
695 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
696 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
697 def consumev2(repo, fp, filecount, filesize): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
698 """Apply the contents from a version 2 streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
699 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
700 Data is read from an object that only needs to provide a ``read(size)`` |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
701 method. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
702 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
703 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
704 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
705 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
706 % (filecount, util.bytecount(filesize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
707 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
708 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
709 start = util.timer() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
710 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
711 _(b'clone'), total=filesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
712 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
713 progress.update(0) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
714 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
715 vfsmap = _makemap(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
716 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
717 with repo.transaction(b'clone'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
718 ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values()) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
719 with nested(*ctxs): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
720 for i in range(filecount): |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
721 src = util.readexactly(fp, 1) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
722 vfs = vfsmap[src] |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
723 namelen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
724 datalen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
725 |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
726 name = util.readexactly(fp, namelen) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
727 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
728 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
729 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
730 b'adding [%s] %s (%s)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
731 % (src, name, util.bytecount(datalen)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
732 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
733 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
734 with vfs(name, b'w') as ofp: |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
735 for chunk in util.filechunkiter(fp, limit=datalen): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
736 progress.increment(step=len(chunk)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
737 ofp.write(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
738 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
739 # force @filecache properties to be reloaded from |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
740 # streamclone-ed file at next access |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
741 repo.invalidate(clearfilecache=True) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
742 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
743 elapsed = util.timer() - start |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
744 if elapsed <= 0: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
745 elapsed = 0.001 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
746 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
747 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
748 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
749 util.bytecount(progress.pos), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
750 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
751 util.bytecount(progress.pos / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
752 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
753 ) |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
754 progress.complete() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
755 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
756 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
757 def applybundlev2(repo, fp, filecount, filesize, requirements): |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
758 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
759 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
760 missingreqs = [r for r in requirements if r not in repo.supported] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
761 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
762 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
763 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
764 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
765 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
766 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
767 consumev2(repo, fp, filecount, filesize) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
768 |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
769 # new requirements = old non-format requirements + |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
770 # new format-related remote requirements |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
771 # requirements from the streamed-in repository |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
772 repo.requirements = set(requirements) | ( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
773 repo.requirements - repo.supportedformats |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
774 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
775 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
776 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
777 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
778 scmutil.writereporequirements(repo) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
779 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
780 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
781 def _copy_files(src_vfs_map, dst_vfs_map, entries, progress): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
782 hardlink = [True] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
783 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
784 def copy_used(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
785 hardlink[0] = False |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
786 progress.topic = _(b'copying') |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
787 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
788 for k, path, size in entries: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
789 src_vfs = src_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
790 dst_vfs = dst_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
791 src_path = src_vfs.join(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
792 dst_path = dst_vfs.join(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
793 dirname = dst_vfs.dirname(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
794 if not dst_vfs.exists(dirname): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
795 dst_vfs.makedirs(dirname) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
796 dst_vfs.register_file(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
797 # XXX we could use the #nb_bytes argument. |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
798 util.copyfile( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
799 src_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
800 dst_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
801 hardlink=hardlink[0], |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
802 no_hardlink_cb=copy_used, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
803 check_fs_hardlink=False, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
804 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
805 progress.increment() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
806 return hardlink[0] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
807 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
808 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
809 def local_copy(src_repo, dest_repo): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
810 """copy all content from one local repository to another |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
811 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
812 This is useful for local clone""" |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
813 src_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
814 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
815 for r in src_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
816 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
817 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
818 dest_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
819 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
820 for r in dest_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
821 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
822 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
823 assert src_store_requirements == dest_store_requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
824 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
825 with dest_repo.lock(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
826 with src_repo.lock(): |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
827 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
828 # bookmark is not integrated to the streaming as it might use the |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
829 # `repo.vfs` and they are too many sentitive data accessible |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
830 # through `repo.vfs` to expose it to streaming clone. |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
831 src_book_vfs = bookmarks.bookmarksvfs(src_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
832 srcbookmarks = src_book_vfs.join(b'bookmarks') |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
833 bm_count = 0 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
834 if os.path.exists(srcbookmarks): |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
835 bm_count = 1 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
836 |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
837 entries, totalfilesize = _v2_walk( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
838 src_repo, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
839 includes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
840 excludes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
841 includeobsmarkers=True, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
842 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
843 src_vfs_map = _makemap(src_repo) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
844 dest_vfs_map = _makemap(dest_repo) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
845 progress = src_repo.ui.makeprogress( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
846 topic=_(b'linking'), |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
847 total=len(entries) + bm_count, |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
848 unit=_(b'files'), |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
849 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
850 # copy files |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
851 # |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
852 # We could copy the full file while the source repository is locked |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
853 # and the other one without the lock. However, in the linking case, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
854 # this would also requires checks that nobody is appending any data |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
855 # to the files while we do the clone, so this is not done yet. We |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
856 # could do this blindly when copying files. |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
857 files = ((k, path, size) for k, path, ftype, size in entries) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
858 hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
859 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
860 # copy bookmarks over |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
861 if bm_count: |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
862 dst_book_vfs = bookmarks.bookmarksvfs(dest_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
863 dstbookmarks = dst_book_vfs.join(b'bookmarks') |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
864 util.copyfile(srcbookmarks, dstbookmarks) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
865 progress.complete() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
866 if hardlink: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
867 msg = b'linked %d files\n' |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
868 else: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
869 msg = b'copied %d files\n' |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
870 src_repo.ui.debug(msg % (len(entries) + bm_count)) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
871 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
872 with dest_repo.transaction(b"localclone") as tr: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
873 dest_repo.store.write(tr) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
874 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
875 # clean up transaction file as they do not make sense |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
876 undo_files = [(dest_repo.svfs, b'undo.backupfiles')] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
877 undo_files.extend(dest_repo.undofiles()) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
878 for undovfs, undofile in undo_files: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
879 try: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
880 undovfs.unlink(undofile) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
881 except OSError as e: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
882 if e.errno != errno.ENOENT: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
883 msg = _(b'error removing %s: %s\n') |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
884 path = undovfs.join(undofile) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
885 e_msg = stringutil.forcebytestr(e) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
886 msg %= (path, e_msg) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
887 dest_repo.ui.warn(msg) |