Mercurial > hg
annotate mercurial/streamclone.py @ 50793:1fbae268ecae
wrapfunction: use sysstr instead of bytes as argument in "sqlitestore"
This is as valid and simpler, it will help us to eventually get ride of
`safehasattr`.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Fri, 03 Feb 2023 04:27:17 +0100 |
parents | 0452af304808 |
children | d718eddf01d9 |
rev | line source |
---|---|
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - producing and consuming streaming repository data |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
9 import contextlib |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
10 import os |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
11 import struct |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
12 |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
13 from .i18n import _ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
14 from .interfaces import repository |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 from . import ( |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
16 bookmarks, |
50523
a6543983b8f4
stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50522
diff
changeset
|
17 bundle2 as bundle2mod, |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
18 cacheutil, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
19 error, |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
20 narrowspec, |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
21 phases, |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
22 pycompat, |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
23 requirements as requirementsmod, |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
24 scmutil, |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
25 store, |
50286
3d0b5760851c
undo-files: move the undo cleanup code in the transaction module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50279
diff
changeset
|
26 transaction, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
27 util, |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 ) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
29 from .revlogutils import ( |
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
30 nodemap, |
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
31 ) |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
33 |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
34 def new_stream_clone_requirements(default_requirements, streamed_requirements): |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
35 """determine the final set of requirement for a new stream clone |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
36 |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
37 this method combine the "default" requirements that a new repository would |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
38 use with the constaint we get from the stream clone content. We keep local |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
39 configuration choice when possible. |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
40 """ |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
41 requirements = set(default_requirements) |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
42 requirements -= requirementsmod.STREAM_FIXED_REQUIREMENTS |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
43 requirements.update(streamed_requirements) |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
44 return requirements |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
45 |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
46 |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
47 def streamed_requirements(repo): |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
48 """the set of requirement the new clone will have to support |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
49 |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
50 This is used for advertising the stream options and to generate the actual |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
51 stream content.""" |
48601
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
52 requiredformats = ( |
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
53 repo.requirements & requirementsmod.STREAM_FIXED_REQUIREMENTS |
baddab229b86
stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48600
diff
changeset
|
54 ) |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
55 return requiredformats |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
56 |
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
57 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
58 def canperformstreamclone(pullop, bundle2=False): |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
59 """Whether it is possible to perform a streaming clone as part of pull. |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
60 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
61 ``bundle2`` will cause the function to consider stream clone through |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
62 bundle2 and only through bundle2. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
63 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
64 Returns a tuple of (supported, requirements). ``supported`` is True if |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
65 streaming clone is supported and False otherwise. ``requirements`` is |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
66 a set of repo requirements from the remote, or ``None`` if stream clone |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
67 isn't supported. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
68 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
69 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
70 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
71 |
50522
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
72 # should we consider streaming clone at all ? |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
73 streamrequested = pullop.streamclonerequested |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
74 # If we don't have a preference, let the server decide for us. This |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
75 # likely only comes into play in LANs. |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
76 if streamrequested is None: |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
77 # The server can advertise whether to prefer streaming clone. |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
78 streamrequested = remote.capable(b'stream-preferred') |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
79 if not streamrequested: |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
80 return False, None |
58e4842fbfc1
stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50521
diff
changeset
|
81 |
50520
0558866957fa
stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50516
diff
changeset
|
82 # Streaming clone only works on an empty destination repository |
0558866957fa
stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50516
diff
changeset
|
83 if len(repo): |
0558866957fa
stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50516
diff
changeset
|
84 return False, None |
0558866957fa
stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50516
diff
changeset
|
85 |
50521
f697af015683
stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50520
diff
changeset
|
86 # Streaming clone only works if all data is being requested. |
f697af015683
stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50520
diff
changeset
|
87 if pullop.heads: |
f697af015683
stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50520
diff
changeset
|
88 return False, None |
f697af015683
stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50520
diff
changeset
|
89 |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
90 bundle2supported = False |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
91 if pullop.canusebundle2: |
50523
a6543983b8f4
stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50522
diff
changeset
|
92 local_caps = bundle2mod.getrepocaps(repo, role=b'client') |
a6543983b8f4
stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50522
diff
changeset
|
93 local_supported = set(local_caps.get(b'stream', [])) |
a6543983b8f4
stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50522
diff
changeset
|
94 remote_supported = set(pullop.remotebundle2caps.get(b'stream', [])) |
a6543983b8f4
stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50522
diff
changeset
|
95 bundle2supported = bool(local_supported & remote_supported) |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
96 # else |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
97 # Server doesn't support bundle2 stream clone or doesn't support |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
98 # the versions we support. Fall back and possibly allow legacy. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
99 |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
100 # Ensures legacy code path uses available bundle2. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
101 if bundle2supported and not bundle2: |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
102 return False, None |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
103 # Ensures bundle2 doesn't try to do a stream clone if it isn't supported. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
104 elif bundle2 and not bundle2supported: |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
105 return False, None |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
106 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
107 # In order for stream clone to work, the client has to support all the |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
108 # requirements advertised by the server. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
109 # |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
110 # The server advertises its requirements via the "stream" and "streamreqs" |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
111 # capability. "stream" (a value-less capability) is advertised if and only |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
112 # if the only requirement is "revlogv1." Else, the "streamreqs" capability |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
113 # is advertised and contains a comma-delimited list of requirements. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
114 requirements = set() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
115 if remote.capable(b'stream'): |
46626
ee91966aec0f
requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents:
45106
diff
changeset
|
116 requirements.add(requirementsmod.REVLOGV1_REQUIREMENT) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
117 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
118 streamreqs = remote.capable(b'streamreqs') |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
119 # This is weird and shouldn't happen with modern servers. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
120 if not streamreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
121 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
122 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
123 b'warning: stream clone requested but server has them ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
124 b'disabled\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
125 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
126 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
127 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
128 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
129 streamreqs = set(streamreqs.split(b',')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
130 # Server requires something we don't support. Bail. |
48670
6d2ddea0721a
stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
131 missingreqs = streamreqs - repo.supported |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
132 if missingreqs: |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
133 pullop.repo.ui.warn( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
134 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
135 b'warning: stream clone requested but client is missing ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
136 b'requirements: %s\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
137 ) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
138 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
139 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
140 pullop.repo.ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
141 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
142 b'(see https://www.mercurial-scm.org/wiki/MissingRequirement ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
143 b'for more information)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
144 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
145 ) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
146 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
147 requirements = streamreqs |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
148 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
149 return True, requirements |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
150 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
151 |
26462
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
152 def maybeperformlegacystreamclone(pullop): |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
153 """Possibly perform a legacy stream clone operation. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
154 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
155 Legacy stream clones are performed as part of pull but before all other |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
156 operations. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
157 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
158 A legacy stream clone will not be performed if a bundle2 stream clone is |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
159 supported. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
160 """ |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
161 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
162 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
163 supported, requirements = canperformstreamclone(pullop) |
26458
362793295640
streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26447
diff
changeset
|
164 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
165 if not supported: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
166 return |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
167 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
168 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
169 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
170 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
171 # Save remote branchmap. We will use it later to speed up branchcache |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
172 # creation. |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
173 rbranchmap = None |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
174 if remote.capable(b'branchmap'): |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
175 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
176 rbranchmap = e.callcommand(b'branchmap', {}).result() |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
177 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
178 repo.ui.status(_(b'streaming all changes\n')) |
26470
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
179 |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
180 with remote.commandexecutor() as e: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
181 fp = e.callcommand(b'stream_out', {}).result() |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
182 |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
183 # TODO strictly speaking, this code should all be inside the context |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
184 # manager because the context manager is supposed to ensure all wire state |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
185 # is flushed when exiting. But the legacy peers don't do this, so it |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
186 # doesn't matter. |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
187 l = fp.readline() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
188 try: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
189 resp = int(l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
190 except ValueError: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
191 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
192 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
193 ) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
194 if resp == 1: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
195 raise error.Abort(_(b'operation forbidden by server')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
196 elif resp == 2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
197 raise error.Abort(_(b'locking the remote repository failed')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
198 elif resp != 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
199 raise error.Abort(_(b'the server sent an unknown error code')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
200 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
201 l = fp.readline() |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
202 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
203 filecount, bytecount = map(int, l.split(b' ', 1)) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
204 except (ValueError, TypeError): |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
205 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
206 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
207 ) |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
208 |
27850
49cfddbf54ba
with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents:
27845
diff
changeset
|
209 with repo.lock(): |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
210 consumev1(repo, fp, filecount, bytecount) |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
211 repo.requirements = new_stream_clone_requirements( |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
212 repo.requirements, |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
213 requirements, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
214 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
215 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
216 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
217 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
218 scmutil.writereporequirements(repo) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
219 nodemap.post_stream_cleanup(repo) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
220 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
221 if rbranchmap: |
41615
328ca3b9e545
branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents:
40494
diff
changeset
|
222 repo._branchcaches.replace(repo, rbranchmap) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
223 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
224 repo.invalidate() |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
225 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
226 |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
227 def allowservergeneration(repo): |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
228 """Whether streaming clones are allowed from the server.""" |
40028
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
229 if repository.REPO_FEATURE_STREAM_CLONE not in repo.features: |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
230 return False |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
231 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
232 if not repo.ui.configbool(b'server', b'uncompressed', untrusted=True): |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
233 return False |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
234 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
235 # The way stream clone works makes it impossible to hide secret changesets. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
236 # So don't allow this by default. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
237 secret = phases.hassecret(repo) |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
238 if secret: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
239 return repo.ui.configbool(b'server', b'uncompressedallowsecret') |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
240 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
241 return True |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
242 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
243 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
244 # This is it's own function so extensions can override it. |
50514
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
245 def _walkstreamfiles(repo, matcher=None, phase=False, obsolescence=False): |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
246 return repo.store.walk(matcher, phase=phase, obsolescence=obsolescence) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
247 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
248 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
249 def generatev1(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
250 """Emit content for version 1 of a streaming clone. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
251 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
252 This returns a 3-tuple of (file count, byte size, data iterator). |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
253 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
254 The data iterator consists of N entries for each file being transferred. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
255 Each file entry starts as a line with the file name and integer size |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
256 delimited by a null byte. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
257 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
258 The raw file data follows. Following the raw file data is the next file |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
259 entry, or EOF. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
260 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
261 When used on the wire protocol, an additional line indicating protocol |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
262 success will be prepended to the stream. This function is not responsible |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
263 for adding it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
264 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
265 This function will obtain a repository lock to ensure a consistent view of |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
266 the store is captured. It therefore may raise LockError. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
267 """ |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
268 entries = [] |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
269 total_bytes = 0 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
270 # Get consistent snapshot of repo, lock during scan. |
27845
7417e1c10253
with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27794
diff
changeset
|
271 with repo.lock(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
272 repo.ui.debug(b'scanning\n') |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50288
diff
changeset
|
273 for entry in _walkstreamfiles(repo): |
50472
9fdc28e21b68
store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50471
diff
changeset
|
274 for f in entry.files(): |
50477
4cbdfab6f812
store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
275 file_size = f.file_size(repo.store.vfs) |
4cbdfab6f812
store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
276 if file_size: |
4cbdfab6f812
store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
277 entries.append((f.unencoded_path, file_size)) |
4cbdfab6f812
store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
278 total_bytes += file_size |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
279 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
280 _test_sync_point_walk_2(repo) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
281 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
282 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
283 b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
284 ) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
285 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
286 svfs = repo.svfs |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
287 debugflag = repo.ui.debugflag |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
288 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
289 def emitrevlogdata(): |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
290 for name, size in entries: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
291 if debugflag: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
292 repo.ui.debug(b'sending %s (%d bytes)\n' % (name, size)) |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
293 # partially encode name over the wire for backwards compat |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
294 yield b'%s\0%d\n' % (store.encodedir(name), size) |
33411
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
295 # auditing at this stage is both pointless (paths are already |
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
296 # trusted by the local repo) and expensive |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
297 with svfs(name, b'rb', auditpath=False) as fp: |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
298 if size <= 65536: |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
299 yield fp.read(size) |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
300 else: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
301 for chunk in util.filechunkiter(fp, limit=size): |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
302 yield chunk |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
303 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
304 return len(entries), total_bytes, emitrevlogdata() |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
305 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
306 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
307 def generatev1wireproto(repo): |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
308 """Emit content for version 1 of streaming clone suitable for the wire. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
309 |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
310 This is the data output from ``generatev1()`` with 2 header lines. The |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
311 first line indicates overall success. The 2nd contains the file count and |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
312 byte size of payload. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
313 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
314 The success line contains "0" for success, "1" for stream generation not |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
315 allowed, and "2" for error locking the repository (possibly indicating |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
316 a permissions error for the server process). |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
317 """ |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
318 if not allowservergeneration(repo): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
319 yield b'1\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
320 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
321 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
322 try: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
323 filecount, bytecount, it = generatev1(repo) |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
324 except error.LockError: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
325 yield b'2\n' |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
326 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
327 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
328 # Indicates successful response. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
329 yield b'0\n' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
330 yield b'%d %d\n' % (filecount, bytecount) |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
331 for chunk in it: |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
332 yield chunk |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
333 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
334 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
335 def generatebundlev1(repo, compression=b'UN'): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
336 """Emit content for version 1 of a stream clone bundle. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
337 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
338 The first 4 bytes of the output ("HGS1") denote this as stream clone |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
339 bundle version 1. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
340 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
341 The next 2 bytes indicate the compression type. Only "UN" is currently |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
342 supported. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
343 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
344 The next 16 bytes are two 64-bit big endian unsigned integers indicating |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
345 file count and byte count, respectively. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
346 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
347 The next 2 bytes is a 16-bit big endian unsigned short declaring the length |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
348 of the requirements string, including a trailing \0. The following N bytes |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
349 are the requirements string, which is ASCII containing a comma-delimited |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
350 list of repo requirements that are needed to support the data. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
351 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
352 The remaining content is the output of ``generatev1()`` (which may be |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
353 compressed in the future). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
354 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
355 Returns a tuple of (requirements, data generator). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
356 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
357 if compression != b'UN': |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
358 raise ValueError(b'we do not support the compression argument yet') |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
359 |
48597
8475a1364909
stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48596
diff
changeset
|
360 requirements = streamed_requirements(repo) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
361 requires = b','.join(sorted(requirements)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
362 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
363 def gen(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
364 yield b'HGS1' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
365 yield compression |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
366 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
367 filecount, bytecount, it = generatev1(repo) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
368 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
369 _(b'writing %d bytes for %d files\n') % (bytecount, filecount) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
370 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
371 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
372 yield struct.pack(b'>QQ', filecount, bytecount) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
373 yield struct.pack(b'>H', len(requires) + 1) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
374 yield requires + b'\0' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
375 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
376 # This is where we'll add compression in the future. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
377 assert compression == b'UN' |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
378 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
379 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
380 _(b'bundle'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
381 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
382 progress.update(0) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
383 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
384 for chunk in it: |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
385 progress.increment(step=len(chunk)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
386 yield chunk |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
387 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
388 progress.complete() |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
389 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
390 return requirements, gen() |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
391 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
392 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
393 def consumev1(repo, fp, filecount, bytecount): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
394 """Apply the contents from version 1 of a streaming clone file handle. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
395 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
396 This takes the output from "stream_out" and applies it to the specified |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
397 repository. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
398 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
399 Like "stream_out," the status line added by the wire protocol is not |
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
400 handled by this function. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
401 """ |
27859
f55a5ace8e69
with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27850
diff
changeset
|
402 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
403 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
404 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
405 % (filecount, util.bytecount(bytecount)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
406 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
407 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
408 _(b'clone'), total=bytecount, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
409 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
410 progress.update(0) |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
411 start = util.timer() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
412 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
413 # TODO: get rid of (potential) inconsistency |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
414 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
415 # If transaction is started and any @filecache property is |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
416 # changed at this point, it causes inconsistency between |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
417 # in-memory cached property and streamclone-ed file on the |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
418 # disk. Nested transaction prevents transaction scope "clone" |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
419 # below from writing in-memory changes out at the end of it, |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
420 # even though in-memory changes are discarded at the end of it |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
421 # regardless of transaction nesting. |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
422 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
423 # But transaction nesting can't be simply prohibited, because |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
424 # nesting occurs also in ordinary case (e.g. enabling |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
425 # clonebundles). |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
426 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
427 with repo.transaction(b'clone'): |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
428 with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
49284
d44e3c45f0e4
py3: replace `pycompat.xrange` by `range`
Manuel Jacob <me@manueljacob.de>
parents:
49190
diff
changeset
|
429 for i in range(filecount): |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
430 # XXX doesn't support '\n' or '\r' in filenames |
50666
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
431 if util.safehasattr(fp, 'readline'): |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
432 l = fp.readline() |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
433 else: |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
434 # inline clonebundles use a chunkbuffer, so no readline |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
435 # --> this should be small anyway, the first line |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
436 # only contains the size of the bundle |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
437 l_buf = [] |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
438 while not (l_buf and l_buf[-1] == b'\n'): |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
439 l_buf.append(fp.read(1)) |
60f9602b413e
clonebundles: add support for inline (streaming) clonebundles
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
50640
diff
changeset
|
440 l = b''.join(l_buf) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
441 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
442 name, size = l.split(b'\0', 1) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
443 size = int(size) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
444 except (ValueError, TypeError): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
445 raise error.ResponseError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
446 _(b'unexpected response from remote server:'), l |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
447 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
448 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
449 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
450 b'adding %s (%s)\n' % (name, util.bytecount(size)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
451 ) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
452 # for backwards compat, name was partially encoded |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
453 path = store.decodedir(name) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
454 with repo.svfs(path, b'w', backgroundclose=True) as ofp: |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
455 for chunk in util.filechunkiter(fp, limit=size): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
456 progress.increment(step=len(chunk)) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
457 ofp.write(chunk) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
458 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
459 # force @filecache properties to be reloaded from |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
460 # streamclone-ed file at next access |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
461 repo.invalidate(clearfilecache=True) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
462 |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
463 elapsed = util.timer() - start |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
464 if elapsed <= 0: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
465 elapsed = 0.001 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
466 progress.complete() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
467 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
468 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
469 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
470 util.bytecount(bytecount), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
471 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
472 util.bytecount(bytecount / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
473 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
474 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
475 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
476 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
477 def readbundle1header(fp): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
478 compression = fp.read(2) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
479 if compression != b'UN': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
480 raise error.Abort( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
481 _( |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
482 b'only uncompressed stream clone bundles are ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
483 b'supported; got %s' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
484 ) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
485 % compression |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
486 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
487 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
488 filecount, bytecount = struct.unpack(b'>QQ', fp.read(16)) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
489 requireslen = struct.unpack(b'>H', fp.read(2))[0] |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
490 requires = fp.read(requireslen) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
491 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
492 if not requires.endswith(b'\0'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
493 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
494 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
495 b'malformed stream clone bundle: ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
496 b'requirements not properly encoded' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
497 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
498 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
499 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
500 requirements = set(requires.rstrip(b'\0').split(b',')) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
501 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
502 return filecount, bytecount, requirements |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
503 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
504 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
505 def applybundlev1(repo, fp): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
506 """Apply the content from a stream clone bundle version 1. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
507 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
508 We assume the 4 byte header has been read and validated and the file handle |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
509 is at the 2 byte compression identifier. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
510 """ |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
511 if len(repo): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
512 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
513 _(b'cannot apply stream clone bundle on non-empty repo') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
514 ) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
515 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
516 filecount, bytecount, requirements = readbundle1header(fp) |
48670
6d2ddea0721a
stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
517 missingreqs = requirements - repo.supported |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
518 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
519 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
520 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
521 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
522 ) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
523 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
524 consumev1(repo, fp, filecount, bytecount) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
525 nodemap.post_stream_cleanup(repo) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
526 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
527 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
528 class streamcloneapplier: |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
529 """Class to manage applying streaming clone bundles. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
530 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
531 We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
532 readers to perform bundle type-specific functionality. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
533 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
534 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
535 def __init__(self, fh): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
536 self._fh = fh |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
537 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
538 def apply(self, repo): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
539 return applybundlev1(repo, self._fh) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
540 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
541 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
542 # type of file to stream |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
543 _fileappend = 0 # append only file |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
544 _filefull = 1 # full snapshot file |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
545 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
546 # Source of the file |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
547 _srcstore = b's' # store (svfs) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
548 _srccache = b'c' # cache (cache) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
549 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
550 # This is it's own function so extensions can override it. |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
551 def _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
552 """list snapshot file from the store""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
553 fnames = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
554 if not repo.publishing(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
555 fnames.append(b'phaseroots') |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
556 return fnames |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
557 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
558 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
559 def _filterfull(entry, copy, vfsmap): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
560 """actually copy the snapshot files""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
561 src, name, ftype, data = entry |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
562 if ftype != _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
563 return entry |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
564 return (src, name, ftype, copy(vfsmap[src].join(name))) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
565 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
566 |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
567 class TempCopyManager: |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
568 """Manage temporary backup of volatile file during stream clone |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
569 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
570 This should be used as a Python context, the copies will be discarded when |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
571 exiting the context. |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
572 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
573 A copy can be done by calling the object on the real path (encoded full |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
574 path) |
49190
4ff4e23de7df
clone: use better names for temp files
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
48946
diff
changeset
|
575 |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
576 The backup path can be retrieved using the __getitem__ protocol, obj[path]. |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
577 On file without backup, it will return the unmodified path. (equivalent to |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
578 `dict.get(x, x)`) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
579 """ |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
580 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
581 def __init__(self): |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
582 self._copies = None |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
583 self._dst_dir = None |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
584 |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
585 def __enter__(self): |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
586 if self._copies is not None: |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
587 msg = "Copies context already open" |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
588 raise error.ProgrammingError(msg) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
589 self._copies = {} |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
590 self._dst_dir = pycompat.mkdtemp(prefix=b'hg-clone-') |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
591 return self |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
592 |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
593 def __call__(self, src): |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
594 """create a backup of the file at src""" |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
595 prefix = os.path.basename(src) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
596 fd, dst = pycompat.mkstemp(prefix=prefix, dir=self._dst_dir) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
597 os.close(fd) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
598 self._copies[src] = dst |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
599 util.copyfiles(src, dst, hardlink=True) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
600 return dst |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
601 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
602 def __getitem__(self, src): |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
603 """return the path to a valid version of `src` |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
604 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
605 If the file has no backup, the path of the file is returned |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
606 unmodified.""" |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
607 return self._copies.get(src, src) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
608 |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
609 def __exit__(self, *args, **kwars): |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
610 """discard all backups""" |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
611 for tmp in self._copies.values(): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
612 util.tryunlink(tmp) |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
613 util.tryrmdir(self._dst_dir) |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
614 self._copies = None |
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
615 self._dst_dir = None |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
616 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
617 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
618 def _makemap(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
619 """make a (src -> vfs) map for the repo""" |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
620 vfsmap = { |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
621 _srcstore: repo.svfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
622 _srccache: repo.cachevfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
623 } |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
624 # we keep repo.vfs out of the on purpose, ther are too many danger there |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
625 # (eg: .hg/hgrc) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
626 assert repo.vfs not in vfsmap.values() |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
627 |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
628 return vfsmap |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
629 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
630 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
631 def _emit2(repo, entries): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
632 """actually emit the stream bundle""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
633 vfsmap = _makemap(repo) |
47502
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
634 # we keep repo.vfs out of the on purpose, ther are too many danger there |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
635 # (eg: .hg/hgrc), |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
636 # |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
637 # this assert is duplicated (from _makemap) as author might think this is |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
638 # fine, while this is really not fine. |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
639 if repo.vfs in vfsmap.values(): |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
640 raise error.ProgrammingError( |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
641 b'repo.vfs must not be added to vfsmap for security reasons' |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
642 ) |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
643 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
644 # translate the vfs one |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
645 entries = [(vfs_key, vfsmap[vfs_key], e) for (vfs_key, e) in entries] |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
646 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
647 max_linkrev = len(repo) |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
648 file_count = totalfilesize = 0 |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
649 # record the expected size of every file |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
650 for k, vfs, e in entries: |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
651 for f in e.files(): |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
652 file_count += 1 |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
653 totalfilesize += f.file_size(vfs) |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
654 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
655 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
656 _(b'bundle'), total=totalfilesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
657 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
658 progress.update(0) |
50630
8c7b04e69894
stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50524
diff
changeset
|
659 with TempCopyManager() as copy, progress: |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
660 # create a copy of volatile files |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
661 for k, vfs, e in entries: |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
662 for f in e.files(): |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
663 if f.is_volatile: |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
664 copy(vfs.join(f.unencoded_path)) |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
665 # the first yield release the lock on the repository |
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
666 yield file_count, totalfilesize |
47788
48f07adbda98
streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47759
diff
changeset
|
667 totalbytecount = 0 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
668 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
669 for src, vfs, e in entries: |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
670 entry_streams = e.get_streams( |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
671 repo=repo, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
672 vfs=vfs, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
673 copies=copy, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
674 max_changeset=max_linkrev, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
675 preserve_file_count=True, |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
676 ) |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
677 for name, stream, size in entry_streams: |
50634
1415e17ea20f
stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50633
diff
changeset
|
678 yield src |
1415e17ea20f
stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50633
diff
changeset
|
679 yield util.uvarintencode(len(name)) |
50639
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
680 yield util.uvarintencode(size) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
681 yield name |
50634
1415e17ea20f
stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50633
diff
changeset
|
682 bytecount = 0 |
50639
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
683 for chunk in stream: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
684 bytecount += len(chunk) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
685 totalbytecount += len(chunk) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
686 progress.update(totalbytecount) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
687 yield chunk |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
688 if bytecount != size: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
689 # Would most likely be caused by a race due to `hg |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
690 # strip` or a revlog split |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
691 msg = _( |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
692 b'clone could only read %d bytes from %s, but ' |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
693 b'expected %d bytes' |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
694 ) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50636
diff
changeset
|
695 raise error.Abort(msg % (bytecount, name, size)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
696 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
697 |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
698 def _emit3(repo, entries): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
699 """actually emit the stream bundle (v3)""" |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
700 vfsmap = _makemap(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
701 # we keep repo.vfs out of the map on purpose, ther are too many dangers |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
702 # there (eg: .hg/hgrc), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
703 # |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
704 # this assert is duplicated (from _makemap) as authors might think this is |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
705 # fine, while this is really not fine. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
706 if repo.vfs in vfsmap.values(): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
707 raise error.ProgrammingError( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
708 b'repo.vfs must not be added to vfsmap for security reasons' |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
709 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
710 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
711 # translate the vfs once |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
712 entries = [(vfs_key, vfsmap[vfs_key], e) for (vfs_key, e) in entries] |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
713 total_entry_count = len(entries) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
714 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
715 max_linkrev = len(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
716 progress = repo.ui.makeprogress( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
717 _(b'bundle'), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
718 total=total_entry_count, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
719 unit=_(b'entry'), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
720 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
721 progress.update(0) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
722 with TempCopyManager() as copy, progress: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
723 # create a copy of volatile files |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
724 for k, vfs, e in entries: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
725 for f in e.files(): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
726 f.file_size(vfs) # record the expected size under lock |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
727 if f.is_volatile: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
728 copy(vfs.join(f.unencoded_path)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
729 # the first yield release the lock on the repository |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
730 yield None |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
731 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
732 yield util.uvarintencode(total_entry_count) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
733 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
734 for src, vfs, e in entries: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
735 entry_streams = e.get_streams( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
736 repo=repo, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
737 vfs=vfs, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
738 copies=copy, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
739 max_changeset=max_linkrev, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
740 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
741 yield util.uvarintencode(len(entry_streams)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
742 for name, stream, size in entry_streams: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
743 yield src |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
744 yield util.uvarintencode(len(name)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
745 yield util.uvarintencode(size) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
746 yield name |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
747 yield from stream |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
748 progress.increment() |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
749 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
750 |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
751 def _test_sync_point_walk_1(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
752 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
753 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
754 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
755 def _test_sync_point_walk_2(repo): |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
756 """a function for synchronisation during tests""" |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
757 |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
758 |
50515
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
759 def _entries_walk(repo, includes, excludes, includeobsmarkers): |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
760 """emit a seris of files information useful to clone a repo |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
761 |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
762 return (vfs-key, entry) iterator |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
763 |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
764 Where `entry` is StoreEntry. (used even for cache entries) |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
765 """ |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
766 assert repo._currentlock(repo._lockref) is not None |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
767 |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
768 matcher = None |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
769 if includes or excludes: |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
770 matcher = narrowspec.match(repo.root, includes, excludes) |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
771 |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
772 phase = not repo.publishing() |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
773 entries = _walkstreamfiles( |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
774 repo, |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
775 matcher, |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
776 phase=phase, |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
777 obsolescence=includeobsmarkers, |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
778 ) |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
779 for entry in entries: |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
780 yield (_srcstore, entry) |
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
781 |
50516
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
782 for name in cacheutil.cachetocopy(repo): |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
783 if repo.cachevfs.exists(name): |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
784 # not really a StoreEntry, but close enough |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
785 entry = store.SimpleStoreEntry( |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
786 entry_path=name, |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
787 is_volatile=True, |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
788 ) |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
789 yield (_srccache, entry) |
43ed1f12b00a
stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50515
diff
changeset
|
790 |
50515
06d580b8f432
stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
791 |
40398
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
792 def generatev2(repo, includes, excludes, includeobsmarkers): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
793 """Emit content for version 2 of a streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
794 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
795 the data stream consists the following entries: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
796 1) A char representing the file destination (eg: store or cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
797 2) A varint containing the length of the filename |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
798 3) A varint containing the length of file data |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
799 4) N bytes containing the filename (the internal, store-agnostic form) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
800 5) N bytes containing the file data |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
801 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
802 Returns a 3-tuple of (file count, file size, data iterator). |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
803 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
804 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
805 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
806 |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
807 repo.ui.debug(b'scanning\n') |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
808 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
809 entries = _entries_walk( |
47444
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
810 repo, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
811 includes=includes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
812 excludes=excludes, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
813 includeobsmarkers=includeobsmarkers, |
2f4ca4807033
streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
814 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
815 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
816 chunks = _emit2(repo, entries) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
817 first = next(chunks) |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
818 file_count, total_file_size = first |
46986
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
819 _test_sync_point_walk_1(repo) |
faa43f09ad98
streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
820 _test_sync_point_walk_2(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
821 |
50635
3416b46320dc
stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50634
diff
changeset
|
822 return file_count, total_file_size, chunks |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
823 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
824 |
50524
58adcabc295f
stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50523
diff
changeset
|
825 def generatev3(repo, includes, excludes, includeobsmarkers): |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
826 """Emit content for version 3 of a streaming clone. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
827 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
828 the data stream consists the following: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
829 1) A varint E containing the number of entries (can be 0), then E entries follow |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
830 2) For each entry: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
831 2.1) The number of files in this entry (can be 0, but typically 1 or 2) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
832 2.2) For each file: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
833 2.2.1) A char representing the file destination (eg: store or cache) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
834 2.2.2) A varint N containing the length of the filename |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
835 2.2.3) A varint M containing the length of file data |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
836 2.2.4) N bytes containing the filename (the internal, store-agnostic form) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
837 2.2.5) M bytes containing the file data |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
838 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
839 Returns the data iterator. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
840 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
841 XXX This format is experimental and subject to change. Here is a |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
842 XXX non-exhaustive list of things this format could do or change: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
843 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
844 - making it easier to write files in parallel |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
845 - holding the lock for a shorter time |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
846 - improving progress information |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
847 - ways to adjust the number of expected entries/files ? |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
848 """ |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
849 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
850 with repo.lock(): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
851 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
852 repo.ui.debug(b'scanning\n') |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
853 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
854 entries = _entries_walk( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
855 repo, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
856 includes=includes, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
857 excludes=excludes, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
858 includeobsmarkers=includeobsmarkers, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
859 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
860 chunks = _emit3(repo, list(entries)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
861 first = next(chunks) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
862 assert first is None |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
863 _test_sync_point_walk_1(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
864 _test_sync_point_walk_2(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
865 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
866 return chunks |
50524
58adcabc295f
stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50523
diff
changeset
|
867 |
58adcabc295f
stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50523
diff
changeset
|
868 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
869 @contextlib.contextmanager |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
870 def nested(*ctxs): |
39757
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
871 this = ctxs[0] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
872 rest = ctxs[1:] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
873 with this: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
874 if rest: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
875 with nested(*rest): |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
876 yield |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
877 else: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
878 yield |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
879 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
880 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
881 def consumev2(repo, fp, filecount, filesize): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
882 """Apply the contents from a version 2 streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
883 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
884 Data is read from an object that only needs to provide a ``read(size)`` |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
885 method. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
886 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
887 with repo.lock(): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
888 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
889 _(b'%d files to transfer, %s of data\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
890 % (filecount, util.bytecount(filesize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
891 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
892 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
893 start = util.timer() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
894 progress = repo.ui.makeprogress( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
895 _(b'clone'), total=filesize, unit=_(b'bytes') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
896 ) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
897 progress.update(0) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
898 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
899 vfsmap = _makemap(repo) |
47502
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
900 # we keep repo.vfs out of the on purpose, ther are too many danger |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
901 # there (eg: .hg/hgrc), |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
902 # |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
903 # this assert is duplicated (from _makemap) as author might think this |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
904 # is fine, while this is really not fine. |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
905 if repo.vfs in vfsmap.values(): |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
906 raise error.ProgrammingError( |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
907 b'repo.vfs must not be added to vfsmap for security reasons' |
65c519661991
stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
908 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
909 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
910 with repo.transaction(b'clone'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
911 ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values()) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
912 with nested(*ctxs): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
913 for i in range(filecount): |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
914 src = util.readexactly(fp, 1) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
915 vfs = vfsmap[src] |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
916 namelen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
917 datalen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
918 |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
919 name = util.readexactly(fp, namelen) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
920 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
921 if repo.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
922 repo.ui.debug( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
923 b'adding [%s] %s (%s)\n' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
924 % (src, name, util.bytecount(datalen)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
925 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
926 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
927 with vfs(name, b'w') as ofp: |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
928 for chunk in util.filechunkiter(fp, limit=datalen): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
929 progress.increment(step=len(chunk)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
930 ofp.write(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
931 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
932 # force @filecache properties to be reloaded from |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
933 # streamclone-ed file at next access |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
934 repo.invalidate(clearfilecache=True) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
935 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
936 elapsed = util.timer() - start |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
937 if elapsed <= 0: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
938 elapsed = 0.001 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
939 repo.ui.status( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
940 _(b'transferred %s in %.1f seconds (%s/sec)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
941 % ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
942 util.bytecount(progress.pos), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
943 elapsed, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
944 util.bytecount(progress.pos / elapsed), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
945 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
946 ) |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
947 progress.complete() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
948 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
949 |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
950 def consumev3(repo, fp): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
951 """Apply the contents from a version 3 streaming clone. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
952 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
953 Data is read from an object that only needs to provide a ``read(size)`` |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
954 method. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
955 """ |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
956 with repo.lock(): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
957 start = util.timer() |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
958 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
959 entrycount = util.uvarintdecodestream(fp) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
960 repo.ui.status(_(b'%d entries to transfer\n') % (entrycount)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
961 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
962 progress = repo.ui.makeprogress( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
963 _(b'clone'), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
964 total=entrycount, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
965 unit=_(b'entries'), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
966 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
967 progress.update(0) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
968 bytes_transferred = 0 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
969 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
970 vfsmap = _makemap(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
971 # we keep repo.vfs out of the on purpose, there are too many dangers |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
972 # there (eg: .hg/hgrc), |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
973 # |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
974 # this assert is duplicated (from _makemap) as authors might think this |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
975 # is fine, while this is really not fine. |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
976 if repo.vfs in vfsmap.values(): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
977 raise error.ProgrammingError( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
978 b'repo.vfs must not be added to vfsmap for security reasons' |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
979 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
980 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
981 with repo.transaction(b'clone'): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
982 ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values()) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
983 with nested(*ctxs): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
984 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
985 for i in range(entrycount): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
986 filecount = util.uvarintdecodestream(fp) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
987 if filecount == 0: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
988 if repo.ui.debugflag: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
989 repo.ui.debug(b'entry with no files [%d]\n' % (i)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
990 for i in range(filecount): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
991 src = util.readexactly(fp, 1) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
992 vfs = vfsmap[src] |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
993 namelen = util.uvarintdecodestream(fp) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
994 datalen = util.uvarintdecodestream(fp) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
995 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
996 name = util.readexactly(fp, namelen) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
997 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
998 if repo.ui.debugflag: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
999 msg = b'adding [%s] %s (%s)\n' |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1000 msg %= (src, name, util.bytecount(datalen)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1001 repo.ui.debug(msg) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1002 bytes_transferred += datalen |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1003 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1004 with vfs(name, b'w') as ofp: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1005 for chunk in util.filechunkiter(fp, limit=datalen): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1006 ofp.write(chunk) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1007 progress.increment(step=1) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1008 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1009 # force @filecache properties to be reloaded from |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1010 # streamclone-ed file at next access |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1011 repo.invalidate(clearfilecache=True) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1012 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1013 elapsed = util.timer() - start |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1014 if elapsed <= 0: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1015 elapsed = 0.001 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1016 msg = _(b'transferred %s in %.1f seconds (%s/sec)\n') |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1017 byte_count = util.bytecount(bytes_transferred) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1018 bytes_sec = util.bytecount(bytes_transferred / elapsed) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1019 msg %= (byte_count, elapsed, bytes_sec) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1020 repo.ui.status(msg) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1021 progress.complete() |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1022 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1023 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
1024 def applybundlev2(repo, fp, filecount, filesize, requirements): |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
1025 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
1026 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
1027 missingreqs = [r for r in requirements if r not in repo.supported] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
1028 if missingreqs: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
1029 raise error.Abort( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43085
diff
changeset
|
1030 _(b'unable to apply stream clone: unsupported format: %s') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1031 % b', '.join(sorted(missingreqs)) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
1032 ) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
1033 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
1034 consumev2(repo, fp, filecount, filesize) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
1035 |
48596
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
1036 repo.requirements = new_stream_clone_requirements( |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
1037 repo.requirements, |
739f2ca3aa3f
stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47878
diff
changeset
|
1038 requirements, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
1039 ) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
1040 repo.svfs.options = localrepo.resolvestorevfsoptions( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
1041 repo.ui, repo.requirements, repo.features |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42813
diff
changeset
|
1042 ) |
45106
a03c177a4679
scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents:
43117
diff
changeset
|
1043 scmutil.writereporequirements(repo) |
48693
de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
48681
diff
changeset
|
1044 nodemap.post_stream_cleanup(repo) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1045 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1046 |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1047 def applybundlev3(repo, fp, requirements): |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1048 from . import localrepo |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1049 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1050 missingreqs = [r for r in requirements if r not in repo.supported] |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1051 if missingreqs: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1052 msg = _(b'unable to apply stream clone: unsupported format: %s') |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1053 msg %= b', '.join(sorted(missingreqs)) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1054 raise error.Abort(msg) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1055 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1056 consumev3(repo, fp) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1057 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1058 repo.requirements = new_stream_clone_requirements( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1059 repo.requirements, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1060 requirements, |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1061 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1062 repo.svfs.options = localrepo.resolvestorevfsoptions( |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1063 repo.ui, repo.requirements, repo.features |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1064 ) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1065 scmutil.writereporequirements(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1066 nodemap.post_stream_cleanup(repo) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1067 |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50666
diff
changeset
|
1068 |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1069 def _copy_files(src_vfs_map, dst_vfs_map, entries, progress): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1070 hardlink = [True] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1071 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1072 def copy_used(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1073 hardlink[0] = False |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1074 progress.topic = _(b'copying') |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1075 |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1076 for k, path in entries: |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1077 src_vfs = src_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1078 dst_vfs = dst_vfs_map[k] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1079 src_path = src_vfs.join(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1080 dst_path = dst_vfs.join(path) |
47871
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
1081 # We cannot use dirname and makedirs of dst_vfs here because the store |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
1082 # encoding confuses them. See issue 6581 for details. |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
1083 dirname = os.path.dirname(dst_path) |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
1084 if not os.path.exists(dirname): |
132525ead0db
clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47788
diff
changeset
|
1085 util.makedirs(dirname) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1086 dst_vfs.register_file(path) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1087 # XXX we could use the #nb_bytes argument. |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1088 util.copyfile( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1089 src_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1090 dst_path, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1091 hardlink=hardlink[0], |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1092 no_hardlink_cb=copy_used, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1093 check_fs_hardlink=False, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1094 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1095 progress.increment() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1096 return hardlink[0] |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1097 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1098 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1099 def local_copy(src_repo, dest_repo): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1100 """copy all content from one local repository to another |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1101 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1102 This is useful for local clone""" |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1103 src_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1104 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1105 for r in src_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1106 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1107 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1108 dest_store_requirements = { |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1109 r |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1110 for r in dest_repo.requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1111 if r not in requirementsmod.WORKING_DIR_REQUIREMENTS |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1112 } |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1113 assert src_store_requirements == dest_store_requirements |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1114 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1115 with dest_repo.lock(): |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1116 with src_repo.lock(): |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1117 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1118 # bookmark is not integrated to the streaming as it might use the |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1119 # `repo.vfs` and they are too many sentitive data accessible |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1120 # through `repo.vfs` to expose it to streaming clone. |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1121 src_book_vfs = bookmarks.bookmarksvfs(src_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1122 srcbookmarks = src_book_vfs.join(b'bookmarks') |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1123 bm_count = 0 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1124 if os.path.exists(srcbookmarks): |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1125 bm_count = 1 |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1126 |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1127 entries = _entries_walk( |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1128 src_repo, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1129 includes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1130 excludes=None, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1131 includeobsmarkers=True, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1132 ) |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1133 entries = list(entries) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1134 src_vfs_map = _makemap(src_repo) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1135 dest_vfs_map = _makemap(dest_repo) |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1136 total_files = sum(len(e[1].files()) for e in entries) + bm_count |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1137 progress = src_repo.ui.makeprogress( |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1138 topic=_(b'linking'), |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1139 total=total_files, |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1140 unit=_(b'files'), |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1141 ) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1142 # copy files |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1143 # |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1144 # We could copy the full file while the source repository is locked |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1145 # and the other one without the lock. However, in the linking case, |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1146 # this would also requires checks that nobody is appending any data |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1147 # to the files while we do the clone, so this is not done yet. We |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1148 # could do this blindly when copying files. |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1149 files = [ |
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1150 (vfs_key, f.unencoded_path) |
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1151 for vfs_key, e in entries |
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1152 for f in e.files() |
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1153 ] |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1154 hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1155 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1156 # copy bookmarks over |
47448
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1157 if bm_count: |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1158 dst_book_vfs = bookmarks.bookmarksvfs(dest_repo) |
d370256636fe
clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47447
diff
changeset
|
1159 dstbookmarks = dst_book_vfs.join(b'bookmarks') |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1160 util.copyfile(srcbookmarks, dstbookmarks) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1161 progress.complete() |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1162 if hardlink: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1163 msg = b'linked %d files\n' |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1164 else: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1165 msg = b'copied %d files\n' |
50633
f2ae815ae34c
local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50630
diff
changeset
|
1166 src_repo.ui.debug(msg % total_files) |
47447
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1167 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1168 with dest_repo.transaction(b"localclone") as tr: |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1169 dest_repo.store.write(tr) |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1170 |
377d8fc20e34
clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47444
diff
changeset
|
1171 # clean up transaction file as they do not make sense |
50288
d89eecf9605e
undo-files: no longer pass the `repo` to `cleanup_undo_files`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50286
diff
changeset
|
1172 transaction.cleanup_undo_files(dest_repo.ui.warn, dest_repo.vfs_map) |