Mercurial > hg
annotate mercurial/streamclone.py @ 48946:642e31cb55f0
py3: use class X: instead of class X(object):
The inheritance from object is implied in Python 3. So this should
be equivalent.
This change was generated via an automated search and replace. So there
may have been some accidental changes.
Differential Revision: https://phab.mercurial-scm.org/D12352
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Mon, 21 Feb 2022 13:08:28 -0700 |
parents | 6000f5b25c9b |
children | 4ff4e23de7df |
rev | line source |
---|---|
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - producing and consuming streaming repository data |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
9 import contextlib |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
10 import errno |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
11 import os |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
12 import struct |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
13 |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
14 from .i18n import _ |
43085
eef9a2d67051
py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
15 from .pycompat import open |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
16 from .interfaces import repository |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 from . import ( |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
18 bookmarks, |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
19 cacheutil, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
20 error, |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
21 narrowspec, |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
22 phases, |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
23 pycompat, |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
24 requirements as requirementsmod, |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
25 scmutil, |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
26 store, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
27 util, |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 ) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
29 from .revlogutils import ( |
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
30 nodemap, |
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
31 ) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
32 from .utils import ( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
33 stringutil, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
34 ) |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
35 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
36 |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
37 def new_stream_clone_requirements(default_requirements, streamed_requirements): |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
38 """determine the final set of requirement for a new stream clone |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
39 |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
40 this method combine the "default" requirements that a new repository would |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
41 use with the constaint we get from the stream clone content. We keep local |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
42 configuration choice when possible. |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
43 """ |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
44 requirements = set(default_requirements) |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
45 requirements -= requirementsmod.STREAM_FIXED_REQUIREMENTS |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
46 requirements.update(streamed_requirements) |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
47 return requirements |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
48 |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
49 |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
50 def streamed_requirements(repo): |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
51 """the set of requirement the new clone will have to support |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
52 |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
53 This is used for advertising the stream options and to generate the actual |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
54 stream content.""" |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
55 requiredformats = ( |
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
56 repo.requirements & requirementsmod.STREAM_FIXED_REQUIREMENTS |
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
57 ) |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
58 return requiredformats |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
59 |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
60 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
61 def canperformstreamclone(pullop, bundle2=False): |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
62 """Whether it is possible to perform a streaming clone as part of pull. |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
63 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
64 ``bundle2`` will cause the function to consider stream clone through |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
65 bundle2 and only through bundle2. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
66 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
67 Returns a tuple of (supported, requirements). ``supported`` is True if |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
68 streaming clone is supported and False otherwise. ``requirements`` is |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
69 a set of repo requirements from the remote, or ``None`` if stream clone |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
70 isn't supported. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
71 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
72 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
73 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
74 |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
75 bundle2supported = False |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
76 if pullop.canusebundle2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
77 if b'v2' in pullop.remotebundle2caps.get(b'stream', []): |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
78 bundle2supported = True |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
79 # else |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
80 # Server doesn't support bundle2 stream clone or doesn't support |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
81 # the versions we support. Fall back and possibly allow legacy. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
82 |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
83 # Ensures legacy code path uses available bundle2. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
84 if bundle2supported and not bundle2: |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
85 return False, None |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
86 # Ensures bundle2 doesn't try to do a stream clone if it isn't supported. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
87 elif bundle2 and not bundle2supported: |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
88 return False, None |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
89 |
26447
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
90 # Streaming clone only works on empty repositories. |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
91 if len(repo): |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
92 return False, None |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
93 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
94 # Streaming clone only works if all data is being requested. |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
95 if pullop.heads: |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
96 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
97 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
98 streamrequested = pullop.streamclonerequested |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
99 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
100 # If we don't have a preference, let the server decide for us. This |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
101 # likely only comes into play in LANs. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
102 if streamrequested is None: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
103 # The server can advertise whether to prefer streaming clone. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
104 streamrequested = remote.capable(b'stream-preferred') |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
105 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
106 if not streamrequested: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
107 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
108 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
109 # In order for stream clone to work, the client has to support all the |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
110 # requirements advertised by the server. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
111 # |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
112 # The server advertises its requirements via the "stream" and "streamreqs" |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
113 # capability. "stream" (a value-less capability) is advertised if and only |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
114 # if the only requirement is "revlogv1." Else, the "streamreqs" capability |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
115 # is advertised and contains a comma-delimited list of requirements. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
116 requirements = set() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
117 if remote.capable(b'stream'): |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
118 requirements.add(requirementsmod.REVLOGV1_REQUIREMENT) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
119 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
120 streamreqs = remote.capable(b'streamreqs') |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
121 # This is weird and shouldn't happen with modern servers. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
122 if not streamreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
123 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
124 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
125 b'warning: stream clone requested but server has them ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
126 b'disabled\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
127 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
128 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
129 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
130 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
131 streamreqs = set(streamreqs.split(b',')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
132 # Server requires something we don't support. Bail. |
48670
6d2ddea0721a
stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
133 missingreqs = streamreqs - repo.supported |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
134 if missingreqs: |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
135 pullop.repo.ui.warn( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
136 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
137 b'warning: stream clone requested but client is missing ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
138 b'requirements: %s\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
139 ) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
140 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
141 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
142 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
143 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
144 b'(see https://www.mercurial-scm.org/wiki/MissingRequirement ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
145 b'for more information)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
146 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
147 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
148 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
149 requirements = streamreqs |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
150 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
151 return True, requirements |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
152 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
153 |
26462
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
154 def maybeperformlegacystreamclone(pullop): |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
155 """Possibly perform a legacy stream clone operation. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
156 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
157 Legacy stream clones are performed as part of pull but before all other |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
158 operations. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
159 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
160 A legacy stream clone will not be performed if a bundle2 stream clone is |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
161 supported. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
162 """ |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
163 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
164 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
165 supported, requirements = canperformstreamclone(pullop) |
26458
362793295640
streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26447
diff
changeset
|
166 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
167 if not supported: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
168 return |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
169 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
170 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
171 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
172 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
173 # Save remote branchmap. We will use it later to speed up branchcache |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
174 # creation. |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
175 rbranchmap = None |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
176 if remote.capable(b'branchmap'): |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
177 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
178 rbranchmap = e.callcommand(b'branchmap', {}).result() |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
179 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
180 repo.ui.status(_(b'streaming all changes\n')) |
26470
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
181 |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
182 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
183 fp = e.callcommand(b'stream_out', {}).result() |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
184 |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
185 # TODO strictly speaking, this code should all be inside the context |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
186 # manager because the context manager is supposed to ensure all wire state |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
187 # is flushed when exiting. But the legacy peers don't do this, so it |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
188 # doesn't matter. |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
189 l = fp.readline() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
190 try: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
191 resp = int(l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
192 except ValueError: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
193 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
194 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
195 ) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
196 if resp == 1: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
197 raise error.Abort(_(b'operation forbidden by server')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
198 elif resp == 2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
199 raise error.Abort(_(b'locking the remote repository failed')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
200 elif resp != 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
201 raise error.Abort(_(b'the server sent an unknown error code')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
202 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
203 l = fp.readline() |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
204 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
205 filecount, bytecount = map(int, l.split(b' ', 1)) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
206 except (ValueError, TypeError): |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
207 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
208 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
209 ) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
210 |
27850
49cfddbf54ba
with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents:
27845
diff
changeset
|
211 with repo.lock(): |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
212 consumev1(repo, fp, filecount, bytecount) |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
213 repo.requirements = new_stream_clone_requirements( |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
214 repo.requirements, |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
215 requirements, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
216 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
217 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
218 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
219 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
220 scmutil.writereporequirements(repo) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
221 nodemap.post_stream_cleanup(repo) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
222 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
223 if rbranchmap: |
41615
328ca3b9e545
branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents:
40494
diff
changeset
|
224 repo._branchcaches.replace(repo, rbranchmap) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
225 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
226 repo.invalidate() |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
227 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
228 |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
229 def allowservergeneration(repo): |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
230 """Whether streaming clones are allowed from the server.""" |
40028
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
231 if repository.REPO_FEATURE_STREAM_CLONE not in repo.features: |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
232 return False |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
233 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
234 if not repo.ui.configbool(b'server', b'uncompressed', untrusted=True): |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
235 return False |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
236 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
237 # The way stream clone works makes it impossible to hide secret changesets. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
238 # So don't allow this by default. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
239 secret = phases.hassecret(repo) |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
240 if secret: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
241 return repo.ui.configbool(b'server', b'uncompressedallowsecret') |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
242 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
243 return True |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
244 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
245 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
246 # This is it's own function so extensions can override it. |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
247 def _walkstreamfiles(repo, matcher=None): |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
248 return repo.store.walk(matcher) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
249 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
250 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
251 def generatev1(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
252 """Emit content for version 1 of a streaming clone. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
253 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
254 This returns a 3-tuple of (file count, byte size, data iterator). |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
255 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
256 The data iterator consists of N entries for each file being transferred. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
257 Each file entry starts as a line with the file name and integer size |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
258 delimited by a null byte. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
259 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
260 The raw file data follows. Following the raw file data is the next file |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
261 entry, or EOF. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
262 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
263 When used on the wire protocol, an additional line indicating protocol |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
264 success will be prepended to the stream. This function is not responsible |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
265 for adding it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
266 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
267 This function will obtain a repository lock to ensure a consistent view of |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
268 the store is captured. It therefore may raise LockError. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
269 """ |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
270 entries = [] |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
271 total_bytes = 0 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
272 # Get consistent snapshot of repo, lock during scan. |
27845
7417e1c10253
with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27794
diff
changeset
|
273 with repo.lock(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
274 repo.ui.debug(b'scanning\n') |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47788
diff
changeset
|
275 for file_type, name, size in _walkstreamfiles(repo): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
276 if size: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
277 entries.append((name, size)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
278 total_bytes += size |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
279 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
280 _test_sync_point_walk_2(repo) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
281 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
282 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
283 b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
284 ) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
285 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
286 svfs = repo.svfs |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
287 debugflag = repo.ui.debugflag |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
288 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
289 def emitrevlogdata(): |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
290 for name, size in entries: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
291 if debugflag: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
292 repo.ui.debug(b'sending %s (%d bytes)\n' % (name, size)) |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
293 # partially encode name over the wire for backwards compat |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
294 yield b'%s\0%d\n' % (store.encodedir(name), size) |
33411
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
295 # auditing at this stage is both pointless (paths are already |
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
296 # trusted by the local repo) and expensive |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
297 with svfs(name, b'rb', auditpath=False) as fp: |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
298 if size <= 65536: |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
299 yield fp.read(size) |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
300 else: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
301 for chunk in util.filechunkiter(fp, limit=size): |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
302 yield chunk |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
303 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
304 return len(entries), total_bytes, emitrevlogdata() |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
305 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
306 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
307 def generatev1wireproto(repo): |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
308 """Emit content for version 1 of streaming clone suitable for the wire. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
309 |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
310 This is the data output from ``generatev1()`` with 2 header lines. The |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
311 first line indicates overall success. The 2nd contains the file count and |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
312 byte size of payload. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
313 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
314 The success line contains "0" for success, "1" for stream generation not |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
315 allowed, and "2" for error locking the repository (possibly indicating |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
316 a permissions error for the server process). |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
317 """ |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
318 if not allowservergeneration(repo): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
319 yield b'1\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
320 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
321 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
322 try: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
323 filecount, bytecount, it = generatev1(repo) |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
324 except error.LockError: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
325 yield b'2\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
326 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
327 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
328 # Indicates successful response. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
329 yield b'0\n' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
330 yield b'%d %d\n' % (filecount, bytecount) |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
331 for chunk in it: |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
332 yield chunk |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
333 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
334 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
335 def generatebundlev1(repo, compression=b'UN'): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
336 """Emit content for version 1 of a stream clone bundle. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
337 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
338 The first 4 bytes of the output ("HGS1") denote this as stream clone |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
339 bundle version 1. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
340 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
341 The next 2 bytes indicate the compression type. Only "UN" is currently |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
342 supported. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
343 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
344 The next 16 bytes are two 64-bit big endian unsigned integers indicating |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
345 file count and byte count, respectively. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
346 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
347 The next 2 bytes is a 16-bit big endian unsigned short declaring the length |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
348 of the requirements string, including a trailing \0. The following N bytes |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
349 are the requirements string, which is ASCII containing a comma-delimited |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
350 list of repo requirements that are needed to support the data. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
351 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
352 The remaining content is the output of ``generatev1()`` (which may be |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
353 compressed in the future). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
354 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
355 Returns a tuple of (requirements, data generator). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
356 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
357 if compression != b'UN': |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
358 raise ValueError(b'we do not support the compression argument yet') |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
359 |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
360 requirements = streamed_requirements(repo) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
361 requires = b','.join(sorted(requirements)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
362 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
363 def gen(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
364 yield b'HGS1' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
365 yield compression |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
366 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
367 filecount, bytecount, it = generatev1(repo) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
368 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
369 _(b'writing %d bytes for %d files\n') % (bytecount, filecount) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
370 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
371 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
372 yield struct.pack(b'>QQ', filecount, bytecount) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
373 yield struct.pack(b'>H', len(requires) + 1) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
374 yield requires + b'\0' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
375 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
376 # This is where we'll add compression in the future. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
377 assert compression == b'UN' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
378 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
379 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
380 _(b'bundle'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
381 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
382 progress.update(0) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
383 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
384 for chunk in it: |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
385 progress.increment(step=len(chunk)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
386 yield chunk |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
387 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
388 progress.complete() |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
389 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
390 return requirements, gen() |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
391 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
392 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
393 def consumev1(repo, fp, filecount, bytecount): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
394 """Apply the contents from version 1 of a streaming clone file handle. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
395 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
396 This takes the output from "stream_out" and applies it to the specified |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
397 repository. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
398 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
399 Like "stream_out," the status line added by the wire protocol is not |
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
400 handled by this function. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
401 """ |
27859
f55a5ace8e69
with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27850
diff
changeset
|
402 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
403 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
404 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
405 % (filecount, util.bytecount(bytecount)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
406 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
407 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
408 _(b'clone'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
409 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
410 progress.update(0) |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
411 start = util.timer() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
412 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
413 # TODO: get rid of (potential) inconsistency |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
414 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
415 # If transaction is started and any @filecache property is |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
416 # changed at this point, it causes inconsistency between |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
417 # in-memory cached property and streamclone-ed file on the |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
418 # disk. Nested transaction prevents transaction scope "clone" |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
419 # below from writing in-memory changes out at the end of it, |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
420 # even though in-memory changes are discarded at the end of it |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
421 # regardless of transaction nesting. |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
422 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
423 # But transaction nesting can't be simply prohibited, because |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
424 # nesting occurs also in ordinary case (e.g. enabling |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
425 # clonebundles). |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
426 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
427 with repo.transaction(b'clone'): |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
428 with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
38783
e7aa113b14f7
global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38374
diff
changeset
|
429 for i in pycompat.xrange(filecount): |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
430 # XXX doesn't support '\n' or '\r' in filenames |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
431 l = fp.readline() |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
432 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
433 name, size = l.split(b'\0', 1) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
434 size = int(size) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
435 except (ValueError, TypeError): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
436 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
437 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
438 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
439 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
440 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
441 b'adding %s (%s)\n' % (name, util.bytecount(size)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
442 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
443 # for backwards compat, name was partially encoded |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
444 path = store.decodedir(name) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
445 with repo.svfs(path, b'w', backgroundclose=True) as ofp: |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
446 for chunk in util.filechunkiter(fp, limit=size): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
447 progress.increment(step=len(chunk)) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
448 ofp.write(chunk) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
449 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
450 # force @filecache properties to be reloaded from |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
451 # streamclone-ed file at next access |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
452 repo.invalidate(clearfilecache=True) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
453 |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
454 elapsed = util.timer() - start |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
455 if elapsed <= 0: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
456 elapsed = 0.001 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
457 progress.complete() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
458 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
459 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
460 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
461 util.bytecount(bytecount), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
462 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
463 util.bytecount(bytecount / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
464 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
465 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
466 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
467 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
468 def readbundle1header(fp): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
469 compression = fp.read(2) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
470 if compression != b'UN': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
471 raise error.Abort( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
472 _( |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
473 b'only uncompressed stream clone bundles are ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
474 b'supported; got %s' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
475 ) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
476 % compression |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
477 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
478 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
479 filecount, bytecount = struct.unpack(b'>QQ', fp.read(16)) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
480 requireslen = struct.unpack(b'>H', fp.read(2))[0] |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
481 requires = fp.read(requireslen) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
482 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
483 if not requires.endswith(b'\0'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
484 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
485 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
486 b'malformed stream clone bundle: ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
487 b'requirements not properly encoded' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
488 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
489 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
490 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
491 requirements = set(requires.rstrip(b'\0').split(b',')) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
492 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
493 return filecount, bytecount, requirements |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
494 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
495 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
496 def applybundlev1(repo, fp): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
497 """Apply the content from a stream clone bundle version 1. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
498 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
499 We assume the 4 byte header has been read and validated and the file handle |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
500 is at the 2 byte compression identifier. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
501 """ |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
502 if len(repo): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
503 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
504 _(b'cannot apply stream clone bundle on non-empty repo') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
505 ) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
506 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
507 filecount, bytecount, requirements = readbundle1header(fp) |
48670
6d2ddea0721a
stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
508 missingreqs = requirements - repo.supported |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
509 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
510 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
511 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
512 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
513 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
514 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
515 consumev1(repo, fp, filecount, bytecount) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
516 nodemap.post_stream_cleanup(repo) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
517 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
518 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
519 class streamcloneapplier: |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
520 """Class to manage applying streaming clone bundles. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
521 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
522 We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
523 readers to perform bundle type-specific functionality. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
524 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
525 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
526 def __init__(self, fh): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
527 self._fh = fh |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
528 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
529 def apply(self, repo): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
530 return applybundlev1(repo, self._fh) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
531 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
532 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
533 # type of file to stream |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
534 _fileappend = 0 # append only file |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
535 _filefull = 1 # full snapshot file |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
536 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
537 # Source of the file |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
538 _srcstore = b's' # store (svfs) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
539 _srccache = b'c' # cache (cache) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
540 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
541 # This is it's own function so extensions can override it. |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
542 def _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
543 """list snapshot file from the store""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
544 fnames = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
545 if not repo.publishing(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
546 fnames.append(b'phaseroots') |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
547 return fnames |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
548 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
549 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
550 def _filterfull(entry, copy, vfsmap): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
551 """actually copy the snapshot files""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
552 src, name, ftype, data = entry |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
553 if ftype != _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
554 return entry |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
555 return (src, name, ftype, copy(vfsmap[src].join(name))) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
556 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
557 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
558 @contextlib.contextmanager |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
559 def maketempcopies(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
560 """return a function to temporary copy file""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
561 files = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
562 try: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
563 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
564 def copy(src): |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
565 fd, dst = pycompat.mkstemp() |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
566 os.close(fd) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
567 files.append(dst) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
568 util.copyfiles(src, dst, hardlink=True) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
569 return dst |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
570 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
571 yield copy |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
572 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
573 for tmp in files: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
574 util.tryunlink(tmp) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
575 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
576 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
577 def _makemap(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
578 """make a (src -> vfs) map for the repo""" |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
579 vfsmap = { |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
580 _srcstore: repo.svfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
581 _srccache: repo.cachevfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
582 } |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
583 # we keep repo.vfs out of the on purpose, ther are too many danger there |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
584 # (eg: .hg/hgrc) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
585 assert repo.vfs not in vfsmap.values() |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
586 |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
587 return vfsmap |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
588 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
589 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
590 def _emit2(repo, entries, totalfilesize): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
591 """actually emit the stream bundle""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
592 vfsmap = _makemap(repo) |
47502
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
593 # we keep repo.vfs out of the on purpose, ther are too many danger there |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
594 # (eg: .hg/hgrc), |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
595 # |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
596 # this assert is duplicated (from _makemap) as author might think this is |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
597 # fine, while this is really not fine. |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
598 if repo.vfs in vfsmap.values(): |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
599 raise error.ProgrammingError( |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
600 b'repo.vfs must not be added to vfsmap for security reasons' |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
601 ) |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
602 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
603 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
604 _(b'bundle'), total=totalfilesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
605 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
606 progress.update(0) |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
607 with maketempcopies() as copy, progress: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
608 # copy is delayed until we are in the try |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
609 entries = [_filterfull(e, copy, vfsmap) for e in entries] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
610 yield None # this release the lock on the repository |
47788
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
611 totalbytecount = 0 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
612 |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
613 for src, name, ftype, data in entries: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
614 vfs = vfsmap[src] |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
615 yield src |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
616 yield util.uvarintencode(len(name)) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
617 if ftype == _fileappend: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
618 fp = vfs(name) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
619 size = data |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
620 elif ftype == _filefull: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
621 fp = open(data, b'rb') |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
622 size = util.fstat(fp).st_size |
47788
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
623 bytecount = 0 |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
624 try: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
625 yield util.uvarintencode(size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
626 yield name |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
627 if size <= 65536: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
628 chunks = (fp.read(size),) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
629 else: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
630 chunks = util.filechunkiter(fp, limit=size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
631 for chunk in chunks: |
47788
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
632 bytecount += len(chunk) |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
633 totalbytecount += len(chunk) |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
634 progress.update(totalbytecount) |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
635 yield chunk |
47788
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
636 if bytecount != size: |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
637 # Would most likely be caused by a race due to `hg strip` or |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
638 # a revlog split |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
639 raise error.Abort( |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
640 _( |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
641 b'clone could only read %d bytes from %s, but ' |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
642 b'expected %d bytes' |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
643 ) |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
644 % (bytecount, name, size) |
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
645 ) |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
646 finally: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
647 fp.close() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
648 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
649 |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
650 def _test_sync_point_walk_1(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
651 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
652 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
653 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
654 def _test_sync_point_walk_2(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
655 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
656 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
657 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
658 def _v2_walk(repo, includes, excludes, includeobsmarkers): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
659 """emit a seris of files information useful to clone a repo |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
660 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
661 return (entries, totalfilesize) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
662 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
663 entries is a list of tuple (vfs-key, file-path, file-type, size) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
664 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
665 - `vfs-key`: is a key to the right vfs to write the file (see _makemap) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
666 - `name`: file path of the file to copy (to be feed to the vfss) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
667 - `file-type`: do this file need to be copied with the source lock ? |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
668 - `size`: the size of the file (or None) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
669 """ |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
670 assert repo._currentlock(repo._lockref) is not None |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
671 entries = [] |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
672 totalfilesize = 0 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
673 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
674 matcher = None |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
675 if includes or excludes: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
676 matcher = narrowspec.match(repo.root, includes, excludes) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
677 |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47788
diff
changeset
|
678 for rl_type, name, size in _walkstreamfiles(repo, matcher): |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
679 if size: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
680 ft = _fileappend |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
681 if rl_type & store.FILEFLAGS_VOLATILE: |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
682 ft = _filefull |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
683 entries.append((_srcstore, name, ft, size)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
684 totalfilesize += size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
685 for name in _walkstreamfullstorefiles(repo): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
686 if repo.svfs.exists(name): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
687 totalfilesize += repo.svfs.lstat(name).st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
688 entries.append((_srcstore, name, _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
689 if includeobsmarkers and repo.svfs.exists(b'obsstore'): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
690 totalfilesize += repo.svfs.lstat(b'obsstore').st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
691 entries.append((_srcstore, b'obsstore', _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
692 for name in cacheutil.cachetocopy(repo): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
693 if repo.cachevfs.exists(name): |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
694 totalfilesize += repo.cachevfs.lstat(name).st_size |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
695 entries.append((_srccache, name, _filefull, None)) |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
696 return entries, totalfilesize |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
697 |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
698 |
40398
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
699 def generatev2(repo, includes, excludes, includeobsmarkers): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
700 """Emit content for version 2 of a streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
701 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
702 the data stream consists the following entries: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
703 1) A char representing the file destination (eg: store or cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
704 2) A varint containing the length of the filename |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
705 3) A varint containing the length of file data |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
706 4) N bytes containing the filename (the internal, store-agnostic form) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
707 5) N bytes containing the file data |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
708 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
709 Returns a 3-tuple of (file count, file size, data iterator). |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
710 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
711 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
712 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
713 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
714 repo.ui.debug(b'scanning\n') |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
715 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
716 entries, totalfilesize = _v2_walk( |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
717 repo, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
718 includes=includes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
719 excludes=excludes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
720 includeobsmarkers=includeobsmarkers, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
721 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
722 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
723 chunks = _emit2(repo, entries, totalfilesize) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
724 first = next(chunks) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
725 assert first is None |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
726 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
727 _test_sync_point_walk_2(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
728 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
729 return len(entries), totalfilesize, chunks |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
730 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
731 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
732 @contextlib.contextmanager |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
733 def nested(*ctxs): |
39757
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
734 this = ctxs[0] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
735 rest = ctxs[1:] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
736 with this: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
737 if rest: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
738 with nested(*rest): |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
739 yield |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
740 else: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
741 yield |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
742 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
743 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
744 def consumev2(repo, fp, filecount, filesize): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
745 """Apply the contents from a version 2 streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
746 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
747 Data is read from an object that only needs to provide a ``read(size)`` |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
748 method. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
749 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
750 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
751 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
752 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
753 % (filecount, util.bytecount(filesize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
754 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
755 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
756 start = util.timer() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
757 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
758 _(b'clone'), total=filesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
759 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
760 progress.update(0) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
761 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
762 vfsmap = _makemap(repo) |
47502
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
763 # we keep repo.vfs out of the on purpose, ther are too many danger |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
764 # there (eg: .hg/hgrc), |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
765 # |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
766 # this assert is duplicated (from _makemap) as author might think this |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
767 # is fine, while this is really not fine. |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
768 if repo.vfs in vfsmap.values(): |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
769 raise error.ProgrammingError( |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
770 b'repo.vfs must not be added to vfsmap for security reasons' |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
771 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
772 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
773 with repo.transaction(b'clone'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
774 ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values()) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
775 with nested(*ctxs): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
776 for i in range(filecount): |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
777 src = util.readexactly(fp, 1) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
778 vfs = vfsmap[src] |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
779 namelen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
780 datalen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
781 |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
782 name = util.readexactly(fp, namelen) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
783 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
784 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
785 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
786 b'adding [%s] %s (%s)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
787 % (src, name, util.bytecount(datalen)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
788 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
789 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
790 with vfs(name, b'w') as ofp: |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
791 for chunk in util.filechunkiter(fp, limit=datalen): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
792 progress.increment(step=len(chunk)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
793 ofp.write(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
794 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
795 # force @filecache properties to be reloaded from |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
796 # streamclone-ed file at next access |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
797 repo.invalidate(clearfilecache=True) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
798 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
799 elapsed = util.timer() - start |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
800 if elapsed <= 0: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
801 elapsed = 0.001 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
802 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
803 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
804 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
805 util.bytecount(progress.pos), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
806 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
807 util.bytecount(progress.pos / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
808 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
809 ) |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
810 progress.complete() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
811 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
812 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
813 def applybundlev2(repo, fp, filecount, filesize, requirements): |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
814 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
815 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
816 missingreqs = [r for r in requirements if r not in repo.supported] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
817 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
818 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
819 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
820 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
821 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
822 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
823 consumev2(repo, fp, filecount, filesize) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
824 |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
825 repo.requirements = new_stream_clone_requirements( |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
826 repo.requirements, |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
827 requirements, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
828 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
829 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
830 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
831 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
832 scmutil.writereporequirements(repo) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
833 nodemap.post_stream_cleanup(repo) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
834 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
835 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
836 def _copy_files(src_vfs_map, dst_vfs_map, entries, progress): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
837 hardlink = [True] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
838 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
839 def copy_used(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
840 hardlink[0] = False |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
841 progress.topic = _(b'copying') |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
842 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
843 for k, path, size in entries: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
844 src_vfs = src_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
845 dst_vfs = dst_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
846 src_path = src_vfs.join(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
847 dst_path = dst_vfs.join(path) |
47871
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
848 # We cannot use dirname and makedirs of dst_vfs here because the store |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
849 # encoding confuses them. See issue 6581 for details. |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
850 dirname = os.path.dirname(dst_path) |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
851 if not os.path.exists(dirname): |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
852 util.makedirs(dirname) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
853 dst_vfs.register_file(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
854 # XXX we could use the #nb_bytes argument. |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
855 util.copyfile( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
856 src_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
857 dst_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
858 hardlink=hardlink[0], |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
859 no_hardlink_cb=copy_used, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
860 check_fs_hardlink=False, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
861 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
862 progress.increment() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
863 return hardlink[0] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
864 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
865 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
866 def local_copy(src_repo, dest_repo): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
867 """copy all content from one local repository to another |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
868 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
869 This is useful for local clone""" |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
870 src_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
871 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
872 for r in src_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
873 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
874 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
875 dest_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
876 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
877 for r in dest_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
878 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
879 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
880 assert src_store_requirements == dest_store_requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
881 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
882 with dest_repo.lock(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
883 with src_repo.lock(): |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
884 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
885 # bookmark is not integrated to the streaming as it might use the |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
886 # `repo.vfs` and they are too many sentitive data accessible |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
887 # through `repo.vfs` to expose it to streaming clone. |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
888 src_book_vfs = bookmarks.bookmarksvfs(src_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
889 srcbookmarks = src_book_vfs.join(b'bookmarks') |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
890 bm_count = 0 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
891 if os.path.exists(srcbookmarks): |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
892 bm_count = 1 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
893 |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
894 entries, totalfilesize = _v2_walk( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
895 src_repo, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
896 includes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
897 excludes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
898 includeobsmarkers=True, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
899 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
900 src_vfs_map = _makemap(src_repo) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
901 dest_vfs_map = _makemap(dest_repo) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
902 progress = src_repo.ui.makeprogress( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
903 topic=_(b'linking'), |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
904 total=len(entries) + bm_count, |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
905 unit=_(b'files'), |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
906 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
907 # copy files |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
908 # |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
909 # We could copy the full file while the source repository is locked |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
910 # and the other one without the lock. However, in the linking case, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
911 # this would also requires checks that nobody is appending any data |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
912 # to the files while we do the clone, so this is not done yet. We |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
913 # could do this blindly when copying files. |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
914 files = ((k, path, size) for k, path, ftype, size in entries) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
915 hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
916 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
917 # copy bookmarks over |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
918 if bm_count: |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
919 dst_book_vfs = bookmarks.bookmarksvfs(dest_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
920 dstbookmarks = dst_book_vfs.join(b'bookmarks') |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
921 util.copyfile(srcbookmarks, dstbookmarks) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
922 progress.complete() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
923 if hardlink: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
924 msg = b'linked %d files\n' |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
925 else: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
926 msg = b'copied %d files\n' |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
927 src_repo.ui.debug(msg % (len(entries) + bm_count)) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
928 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
929 with dest_repo.transaction(b"localclone") as tr: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
930 dest_repo.store.write(tr) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
931 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
932 # clean up transaction file as they do not make sense |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
933 undo_files = [(dest_repo.svfs, b'undo.backupfiles')] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
934 undo_files.extend(dest_repo.undofiles()) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
935 for undovfs, undofile in undo_files: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
936 try: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
937 undovfs.unlink(undofile) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
938 except OSError as e: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
939 if e.errno != errno.ENOENT: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
940 msg = _(b'error removing %s: %s\n') |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
941 path = undovfs.join(undofile) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
942 e_msg = stringutil.forcebytestr(e) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
943 msg %= (path, e_msg) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
944 dest_repo.ui.warn(msg) |