Mercurial > hg
annotate mercurial/streamclone.py @ 41731:92055d539e49
tests: conditionalize msys path mangling in test-bundle.t
This broke in 252cc56c9ff6 when the variables were printed one per line. The
only reason I can think of is that MSYS knew the former string couldn't be a
list of paths, but thinks this form is.
author | Matt Harbison <matt_harbison@yahoo.com> |
---|---|
date | Sat, 16 Feb 2019 22:03:58 -0500 |
parents | 328ca3b9e545 |
children | 268662aac075 |
rev | line source |
---|---|
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - producing and consuming streaming repository data |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
10 import contextlib |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
11 import os |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
12 import struct |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
13 |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
14 from .i18n import _ |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 from . import ( |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
16 cacheutil, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
17 error, |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
18 narrowspec, |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
19 phases, |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
20 pycompat, |
40028
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
21 repository, |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
22 store, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
23 util, |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 ) |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
26 def canperformstreamclone(pullop, bundle2=False): |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
27 """Whether it is possible to perform a streaming clone as part of pull. |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
28 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
29 ``bundle2`` will cause the function to consider stream clone through |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
30 bundle2 and only through bundle2. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
31 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
32 Returns a tuple of (supported, requirements). ``supported`` is True if |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
33 streaming clone is supported and False otherwise. ``requirements`` is |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
34 a set of repo requirements from the remote, or ``None`` if stream clone |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
35 isn't supported. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
36 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
37 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
38 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
39 |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
40 bundle2supported = False |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
41 if pullop.canusebundle2: |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
42 if 'v2' in pullop.remotebundle2caps.get('stream', []): |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
43 bundle2supported = True |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
44 # else |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
45 # Server doesn't support bundle2 stream clone or doesn't support |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
46 # the versions we support. Fall back and possibly allow legacy. |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
47 |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
48 # Ensures legacy code path uses available bundle2. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
49 if bundle2supported and not bundle2: |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
50 return False, None |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
51 # Ensures bundle2 doesn't try to do a stream clone if it isn't supported. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
52 elif bundle2 and not bundle2supported: |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
53 return False, None |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
54 |
26447
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
55 # Streaming clone only works on empty repositories. |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
56 if len(repo): |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
57 return False, None |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
58 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
59 # Streaming clone only works if all data is being requested. |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
60 if pullop.heads: |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
61 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
62 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
63 streamrequested = pullop.streamclonerequested |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
64 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
65 # If we don't have a preference, let the server decide for us. This |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
66 # likely only comes into play in LANs. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
67 if streamrequested is None: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
68 # The server can advertise whether to prefer streaming clone. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
69 streamrequested = remote.capable('stream-preferred') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
70 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
71 if not streamrequested: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
72 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
73 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
74 # In order for stream clone to work, the client has to support all the |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
75 # requirements advertised by the server. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
76 # |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
77 # The server advertises its requirements via the "stream" and "streamreqs" |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
78 # capability. "stream" (a value-less capability) is advertised if and only |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
79 # if the only requirement is "revlogv1." Else, the "streamreqs" capability |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
80 # is advertised and contains a comma-delimited list of requirements. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
81 requirements = set() |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
82 if remote.capable('stream'): |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
83 requirements.add('revlogv1') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
84 else: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
85 streamreqs = remote.capable('streamreqs') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
86 # This is weird and shouldn't happen with modern servers. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
87 if not streamreqs: |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
88 pullop.repo.ui.warn(_( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
89 'warning: stream clone requested but server has them ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
90 'disabled\n')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
91 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
92 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
93 streamreqs = set(streamreqs.split(',')) |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
94 # Server requires something we don't support. Bail. |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
95 missingreqs = streamreqs - repo.supportedformats |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
96 if missingreqs: |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
97 pullop.repo.ui.warn(_( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
98 'warning: stream clone requested but client is missing ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
99 'requirements: %s\n') % ', '.join(sorted(missingreqs))) |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
100 pullop.repo.ui.warn( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
101 _('(see https://www.mercurial-scm.org/wiki/MissingRequirement ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
102 'for more information)\n')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
103 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
104 requirements = streamreqs |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
105 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
106 return True, requirements |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
107 |
26462
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
108 def maybeperformlegacystreamclone(pullop): |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
109 """Possibly perform a legacy stream clone operation. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
110 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
111 Legacy stream clones are performed as part of pull but before all other |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
112 operations. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
113 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
114 A legacy stream clone will not be performed if a bundle2 stream clone is |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
115 supported. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
116 """ |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
117 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
118 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
119 supported, requirements = canperformstreamclone(pullop) |
26458
362793295640
streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26447
diff
changeset
|
120 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
121 if not supported: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
122 return |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
123 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
124 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
125 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
126 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
127 # Save remote branchmap. We will use it later to speed up branchcache |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
128 # creation. |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
129 rbranchmap = None |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
130 if remote.capable('branchmap'): |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
131 with remote.commandexecutor() as e: |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
132 rbranchmap = e.callcommand('branchmap', {}).result() |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
133 |
26470
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
134 repo.ui.status(_('streaming all changes\n')) |
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
135 |
37638
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
136 with remote.commandexecutor() as e: |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
137 fp = e.callcommand('stream_out', {}).result() |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
138 |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
139 # TODO strictly speaking, this code should all be inside the context |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
140 # manager because the context manager is supposed to ensure all wire state |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
141 # is flushed when exiting. But the legacy peers don't do this, so it |
65b86ee69383
streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35804
diff
changeset
|
142 # doesn't matter. |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
143 l = fp.readline() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
144 try: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
145 resp = int(l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
146 except ValueError: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
147 raise error.ResponseError( |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
148 _('unexpected response from remote server:'), l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
149 if resp == 1: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
150 raise error.Abort(_('operation forbidden by server')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
151 elif resp == 2: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
152 raise error.Abort(_('locking the remote repository failed')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
153 elif resp != 0: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
154 raise error.Abort(_('the server sent an unknown error code')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
155 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
156 l = fp.readline() |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
157 try: |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
158 filecount, bytecount = map(int, l.split(' ', 1)) |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
159 except (ValueError, TypeError): |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
160 raise error.ResponseError( |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
161 _('unexpected response from remote server:'), l) |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
162 |
27850
49cfddbf54ba
with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents:
27845
diff
changeset
|
163 with repo.lock(): |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
164 consumev1(repo, fp, filecount, bytecount) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
165 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
166 # new requirements = old non-format requirements + |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
167 # new format-related remote requirements |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
168 # requirements from the streamed-in repository |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
169 repo.requirements = requirements | ( |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
170 repo.requirements - repo.supportedformats) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
171 repo.svfs.options = localrepo.resolvestorevfsoptions( |
39850
d89d5bc06eaa
localrepo: define "features" on repository instances (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39757
diff
changeset
|
172 repo.ui, repo.requirements, repo.features) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
173 repo._writerequirements() |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
174 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
175 if rbranchmap: |
41615
328ca3b9e545
branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents:
40494
diff
changeset
|
176 repo._branchcaches.replace(repo, rbranchmap) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
177 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
178 repo.invalidate() |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
179 |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
180 def allowservergeneration(repo): |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
181 """Whether streaming clones are allowed from the server.""" |
40028
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
182 if repository.REPO_FEATURE_STREAM_CLONE not in repo.features: |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
183 return False |
51f10e6d66c7
streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39850
diff
changeset
|
184 |
33499
0407a51b9d8c
codemod: register core configitems using a script
Jun Wu <quark@fb.com>
parents:
33411
diff
changeset
|
185 if not repo.ui.configbool('server', 'uncompressed', untrusted=True): |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
186 return False |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
187 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
188 # The way stream clone works makes it impossible to hide secret changesets. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
189 # So don't allow this by default. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
190 secret = phases.hassecret(repo) |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
191 if secret: |
33226
b045344fe35e
configitems: register the 'server.uncompressedallowsecret' config
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
32744
diff
changeset
|
192 return repo.ui.configbool('server', 'uncompressedallowsecret') |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
193 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
194 return True |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
195 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
196 # This is it's own function so extensions can override it. |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
197 def _walkstreamfiles(repo, matcher=None): |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
198 return repo.store.walk(matcher) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
199 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
200 def generatev1(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
201 """Emit content for version 1 of a streaming clone. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
202 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
203 This returns a 3-tuple of (file count, byte size, data iterator). |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
204 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
205 The data iterator consists of N entries for each file being transferred. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
206 Each file entry starts as a line with the file name and integer size |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
207 delimited by a null byte. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
208 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
209 The raw file data follows. Following the raw file data is the next file |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
210 entry, or EOF. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
211 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
212 When used on the wire protocol, an additional line indicating protocol |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
213 success will be prepended to the stream. This function is not responsible |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
214 for adding it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
215 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
216 This function will obtain a repository lock to ensure a consistent view of |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
217 the store is captured. It therefore may raise LockError. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
218 """ |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
219 entries = [] |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
220 total_bytes = 0 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
221 # Get consistent snapshot of repo, lock during scan. |
27845
7417e1c10253
with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27794
diff
changeset
|
222 with repo.lock(): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
223 repo.ui.debug('scanning\n') |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
224 for name, ename, size in _walkstreamfiles(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
225 if size: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
226 entries.append((name, size)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
227 total_bytes += size |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
228 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
229 repo.ui.debug('%d files, %d bytes to transfer\n' % |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
230 (len(entries), total_bytes)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
231 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
232 svfs = repo.svfs |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
233 debugflag = repo.ui.debugflag |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
234 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
235 def emitrevlogdata(): |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
236 for name, size in entries: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
237 if debugflag: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
238 repo.ui.debug('sending %s (%d bytes)\n' % (name, size)) |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
239 # partially encode name over the wire for backwards compat |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
240 yield '%s\0%d\n' % (store.encodedir(name), size) |
33411
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
241 # auditing at this stage is both pointless (paths are already |
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
242 # trusted by the local repo) and expensive |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
243 with svfs(name, 'rb', auditpath=False) as fp: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
244 if size <= 65536: |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
245 yield fp.read(size) |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
246 else: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
247 for chunk in util.filechunkiter(fp, limit=size): |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
248 yield chunk |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
249 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
250 return len(entries), total_bytes, emitrevlogdata() |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
251 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
252 def generatev1wireproto(repo): |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
253 """Emit content for version 1 of streaming clone suitable for the wire. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
254 |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
255 This is the data output from ``generatev1()`` with 2 header lines. The |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
256 first line indicates overall success. The 2nd contains the file count and |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
257 byte size of payload. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
258 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
259 The success line contains "0" for success, "1" for stream generation not |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
260 allowed, and "2" for error locking the repository (possibly indicating |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
261 a permissions error for the server process). |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
262 """ |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
263 if not allowservergeneration(repo): |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
264 yield '1\n' |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
265 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
266 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
267 try: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
268 filecount, bytecount, it = generatev1(repo) |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
269 except error.LockError: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
270 yield '2\n' |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
271 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
272 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
273 # Indicates successful response. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
274 yield '0\n' |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
275 yield '%d %d\n' % (filecount, bytecount) |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
276 for chunk in it: |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
277 yield chunk |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
278 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
279 def generatebundlev1(repo, compression='UN'): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
280 """Emit content for version 1 of a stream clone bundle. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
281 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
282 The first 4 bytes of the output ("HGS1") denote this as stream clone |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
283 bundle version 1. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
284 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
285 The next 2 bytes indicate the compression type. Only "UN" is currently |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
286 supported. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
287 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
288 The next 16 bytes are two 64-bit big endian unsigned integers indicating |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
289 file count and byte count, respectively. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
290 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
291 The next 2 bytes is a 16-bit big endian unsigned short declaring the length |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
292 of the requirements string, including a trailing \0. The following N bytes |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
293 are the requirements string, which is ASCII containing a comma-delimited |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
294 list of repo requirements that are needed to support the data. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
295 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
296 The remaining content is the output of ``generatev1()`` (which may be |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
297 compressed in the future). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
298 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
299 Returns a tuple of (requirements, data generator). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
300 """ |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
301 if compression != 'UN': |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
302 raise ValueError('we do not support the compression argument yet') |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
303 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
304 requirements = repo.requirements & repo.supportedformats |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
305 requires = ','.join(sorted(requirements)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
306 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
307 def gen(): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
308 yield 'HGS1' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
309 yield compression |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
310 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
311 filecount, bytecount, it = generatev1(repo) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
312 repo.ui.status(_('writing %d bytes for %d files\n') % |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
313 (bytecount, filecount)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
314 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
315 yield struct.pack('>QQ', filecount, bytecount) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
316 yield struct.pack('>H', len(requires) + 1) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
317 yield requires + '\0' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
318 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
319 # This is where we'll add compression in the future. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
320 assert compression == 'UN' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
321 |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
322 progress = repo.ui.makeprogress(_('bundle'), total=bytecount, |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
323 unit=_('bytes')) |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
324 progress.update(0) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
325 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
326 for chunk in it: |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
327 progress.increment(step=len(chunk)) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
328 yield chunk |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
329 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
330 progress.complete() |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
331 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
332 return requirements, gen() |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
333 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
334 def consumev1(repo, fp, filecount, bytecount): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
335 """Apply the contents from version 1 of a streaming clone file handle. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
336 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
337 This takes the output from "stream_out" and applies it to the specified |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
338 repository. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
339 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
340 Like "stream_out," the status line added by the wire protocol is not |
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
341 handled by this function. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
342 """ |
27859
f55a5ace8e69
with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27850
diff
changeset
|
343 with repo.lock(): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
344 repo.ui.status(_('%d files to transfer, %s of data\n') % |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
345 (filecount, util.bytecount(bytecount))) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
346 progress = repo.ui.makeprogress(_('clone'), total=bytecount, |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
347 unit=_('bytes')) |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
348 progress.update(0) |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
349 start = util.timer() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
350 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
351 # TODO: get rid of (potential) inconsistency |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
352 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
353 # If transaction is started and any @filecache property is |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
354 # changed at this point, it causes inconsistency between |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
355 # in-memory cached property and streamclone-ed file on the |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
356 # disk. Nested transaction prevents transaction scope "clone" |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
357 # below from writing in-memory changes out at the end of it, |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
358 # even though in-memory changes are discarded at the end of it |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
359 # regardless of transaction nesting. |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
360 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
361 # But transaction nesting can't be simply prohibited, because |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
362 # nesting occurs also in ordinary case (e.g. enabling |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
363 # clonebundles). |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
364 |
27870
ed9950ba091e
with: use context manager for transaction in consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27859
diff
changeset
|
365 with repo.transaction('clone'): |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
366 with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
38783
e7aa113b14f7
global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38374
diff
changeset
|
367 for i in pycompat.xrange(filecount): |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
368 # XXX doesn't support '\n' or '\r' in filenames |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
369 l = fp.readline() |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
370 try: |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
371 name, size = l.split('\0', 1) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
372 size = int(size) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
373 except (ValueError, TypeError): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
374 raise error.ResponseError( |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
375 _('unexpected response from remote server:'), l) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
376 if repo.ui.debugflag: |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
377 repo.ui.debug('adding %s (%s)\n' % |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
378 (name, util.bytecount(size))) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
379 # for backwards compat, name was partially encoded |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
380 path = store.decodedir(name) |
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
381 with repo.svfs(path, 'w', backgroundclose=True) as ofp: |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
382 for chunk in util.filechunkiter(fp, limit=size): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
383 progress.increment(step=len(chunk)) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
384 ofp.write(chunk) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
385 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
386 # force @filecache properties to be reloaded from |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
387 # streamclone-ed file at next access |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
388 repo.invalidate(clearfilecache=True) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
389 |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
390 elapsed = util.timer() - start |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
391 if elapsed <= 0: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
392 elapsed = 0.001 |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
393 progress.complete() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
394 repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
395 (util.bytecount(bytecount), elapsed, |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
396 util.bytecount(bytecount / elapsed))) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
397 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
398 def readbundle1header(fp): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
399 compression = fp.read(2) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
400 if compression != 'UN': |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
401 raise error.Abort(_('only uncompressed stream clone bundles are ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
402 'supported; got %s') % compression) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
403 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
404 filecount, bytecount = struct.unpack('>QQ', fp.read(16)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
405 requireslen = struct.unpack('>H', fp.read(2))[0] |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
406 requires = fp.read(requireslen) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
407 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
408 if not requires.endswith('\0'): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
409 raise error.Abort(_('malformed stream clone bundle: ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
410 'requirements not properly encoded')) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
411 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
412 requirements = set(requires.rstrip('\0').split(',')) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
413 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
414 return filecount, bytecount, requirements |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
415 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
416 def applybundlev1(repo, fp): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
417 """Apply the content from a stream clone bundle version 1. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
418 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
419 We assume the 4 byte header has been read and validated and the file handle |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
420 is at the 2 byte compression identifier. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
421 """ |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
422 if len(repo): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
423 raise error.Abort(_('cannot apply stream clone bundle on non-empty ' |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
424 'repo')) |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
425 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
426 filecount, bytecount, requirements = readbundle1header(fp) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
427 missingreqs = requirements - repo.supportedformats |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
428 if missingreqs: |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
429 raise error.Abort(_('unable to apply stream clone: ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
430 'unsupported format: %s') % |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
431 ', '.join(sorted(missingreqs))) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
432 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
433 consumev1(repo, fp, filecount, bytecount) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
434 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
435 class streamcloneapplier(object): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
436 """Class to manage applying streaming clone bundles. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
437 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
438 We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
439 readers to perform bundle type-specific functionality. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
440 """ |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
441 def __init__(self, fh): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
442 self._fh = fh |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
443 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
444 def apply(self, repo): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
445 return applybundlev1(repo, self._fh) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
446 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
447 # type of file to stream |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
448 _fileappend = 0 # append only file |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
449 _filefull = 1 # full snapshot file |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
450 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
451 # Source of the file |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
452 _srcstore = 's' # store (svfs) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
453 _srccache = 'c' # cache (cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
454 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
455 # This is it's own function so extensions can override it. |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
456 def _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
457 """list snapshot file from the store""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
458 fnames = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
459 if not repo.publishing(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
460 fnames.append('phaseroots') |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
461 return fnames |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
462 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
463 def _filterfull(entry, copy, vfsmap): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
464 """actually copy the snapshot files""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
465 src, name, ftype, data = entry |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
466 if ftype != _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
467 return entry |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
468 return (src, name, ftype, copy(vfsmap[src].join(name))) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
469 |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
470 @contextlib.contextmanager |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
471 def maketempcopies(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
472 """return a function to temporary copy file""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
473 files = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
474 try: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
475 def copy(src): |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37638
diff
changeset
|
476 fd, dst = pycompat.mkstemp() |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
477 os.close(fd) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
478 files.append(dst) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
479 util.copyfiles(src, dst, hardlink=True) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
480 return dst |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
481 yield copy |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
482 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
483 for tmp in files: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
484 util.tryunlink(tmp) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
485 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
486 def _makemap(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
487 """make a (src -> vfs) map for the repo""" |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
488 vfsmap = { |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
489 _srcstore: repo.svfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
490 _srccache: repo.cachevfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
491 } |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
492 # we keep repo.vfs out of the on purpose, ther are too many danger there |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
493 # (eg: .hg/hgrc) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
494 assert repo.vfs not in vfsmap.values() |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
495 |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
496 return vfsmap |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
497 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
498 def _emit2(repo, entries, totalfilesize): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
499 """actually emit the stream bundle""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
500 vfsmap = _makemap(repo) |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
501 progress = repo.ui.makeprogress(_('bundle'), total=totalfilesize, |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
502 unit=_('bytes')) |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
503 progress.update(0) |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
504 with maketempcopies() as copy, progress: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
505 # copy is delayed until we are in the try |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
506 entries = [_filterfull(e, copy, vfsmap) for e in entries] |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
507 yield None # this release the lock on the repository |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
508 seen = 0 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
509 |
38374
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
510 for src, name, ftype, data in entries: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
511 vfs = vfsmap[src] |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
512 yield src |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
513 yield util.uvarintencode(len(name)) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
514 if ftype == _fileappend: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
515 fp = vfs(name) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
516 size = data |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
517 elif ftype == _filefull: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
518 fp = open(data, 'rb') |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
519 size = util.fstat(fp).st_size |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
520 try: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
521 yield util.uvarintencode(size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
522 yield name |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
523 if size <= 65536: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
524 chunks = (fp.read(size),) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
525 else: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
526 chunks = util.filechunkiter(fp, limit=size) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
527 for chunk in chunks: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
528 seen += len(chunk) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
529 progress.update(seen) |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
530 yield chunk |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
531 finally: |
800f5a2c869e
progress: make the progress helper a context manager
Martin von Zweigbergk <martinvonz@google.com>
parents:
38373
diff
changeset
|
532 fp.close() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
533 |
40398
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
534 def generatev2(repo, includes, excludes, includeobsmarkers): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
535 """Emit content for version 2 of a streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
536 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
537 the data stream consists the following entries: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
538 1) A char representing the file destination (eg: store or cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
539 2) A varint containing the length of the filename |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
540 3) A varint containing the length of file data |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
541 4) N bytes containing the filename (the internal, store-agnostic form) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
542 5) N bytes containing the file data |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
543 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
544 Returns a 3-tuple of (file count, file size, data iterator). |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
545 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
546 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
547 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
548 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
549 entries = [] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
550 totalfilesize = 0 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
551 |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
552 matcher = None |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
553 if includes or excludes: |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
554 matcher = narrowspec.match(repo.root, includes, excludes) |
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
555 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
556 repo.ui.debug('scanning\n') |
40339
f0e8f27768eb
streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40338
diff
changeset
|
557 for name, ename, size in _walkstreamfiles(repo, matcher): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
558 if size: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
559 entries.append((_srcstore, name, _fileappend, size)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
560 totalfilesize += size |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
561 for name in _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
562 if repo.svfs.exists(name): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
563 totalfilesize += repo.svfs.lstat(name).st_size |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
564 entries.append((_srcstore, name, _filefull, None)) |
40398
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
565 if includeobsmarkers and repo.svfs.exists('obsstore'): |
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
566 totalfilesize += repo.svfs.lstat('obsstore').st_size |
0ac794e0e285
streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents:
40339
diff
changeset
|
567 entries.append((_srcstore, 'obsstore', _filefull, None)) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
568 for name in cacheutil.cachetocopy(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
569 if repo.cachevfs.exists(name): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
570 totalfilesize += repo.cachevfs.lstat(name).st_size |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
571 entries.append((_srccache, name, _filefull, None)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
572 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
573 chunks = _emit2(repo, entries, totalfilesize) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
574 first = next(chunks) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
575 assert first is None |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
576 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
577 return len(entries), totalfilesize, chunks |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
578 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
579 @contextlib.contextmanager |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
580 def nested(*ctxs): |
39757
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
581 this = ctxs[0] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
582 rest = ctxs[1:] |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
583 with this: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
584 if rest: |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
585 with nested(*rest): |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
586 yield |
97f2992c26f6
streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents:
39700
diff
changeset
|
587 else: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
588 yield |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
589 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
590 def consumev2(repo, fp, filecount, filesize): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
591 """Apply the contents from a version 2 streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
592 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
593 Data is read from an object that only needs to provide a ``read(size)`` |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
594 method. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
595 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
596 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
597 repo.ui.status(_('%d files to transfer, %s of data\n') % |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
598 (filecount, util.bytecount(filesize))) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
599 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
600 start = util.timer() |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
601 progress = repo.ui.makeprogress(_('clone'), total=filesize, |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
602 unit=_('bytes')) |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
603 progress.update(0) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
604 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
605 vfsmap = _makemap(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
606 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
607 with repo.transaction('clone'): |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
608 ctxs = (vfs.backgroundclosing(repo.ui) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
609 for vfs in vfsmap.values()) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
610 with nested(*ctxs): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
611 for i in range(filecount): |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
612 src = util.readexactly(fp, 1) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
613 vfs = vfsmap[src] |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
614 namelen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
615 datalen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
616 |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
617 name = util.readexactly(fp, namelen) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
618 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
619 if repo.ui.debugflag: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
620 repo.ui.debug('adding [%s] %s (%s)\n' % |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
621 (src, name, util.bytecount(datalen))) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
622 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
623 with vfs(name, 'w') as ofp: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
624 for chunk in util.filechunkiter(fp, limit=datalen): |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
625 progress.increment(step=len(chunk)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
626 ofp.write(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
627 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
628 # force @filecache properties to be reloaded from |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
629 # streamclone-ed file at next access |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
630 repo.invalidate(clearfilecache=True) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
631 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
632 elapsed = util.timer() - start |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
633 if elapsed <= 0: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
634 elapsed = 0.001 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
635 repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % |
38349
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
636 (util.bytecount(progress.pos), elapsed, |
e59eaf51cc0d
streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38164
diff
changeset
|
637 util.bytecount(progress.pos / elapsed))) |
38373
ef692614e601
progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents:
38360
diff
changeset
|
638 progress.complete() |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
639 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
640 def applybundlev2(repo, fp, filecount, filesize, requirements): |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
641 from . import localrepo |
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
642 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
643 missingreqs = [r for r in requirements if r not in repo.supported] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
644 if missingreqs: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
645 raise error.Abort(_('unable to apply stream clone: ' |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
646 'unsupported format: %s') % |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
647 ', '.join(sorted(missingreqs))) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
648 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
649 consumev2(repo, fp, filecount, filesize) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
650 |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
651 # new requirements = old non-format requirements + |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
652 # new format-related remote requirements |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
653 # requirements from the streamed-in repository |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
654 repo.requirements = set(requirements) | ( |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
655 repo.requirements - repo.supportedformats) |
39700
b10d145837bc
localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38783
diff
changeset
|
656 repo.svfs.options = localrepo.resolvestorevfsoptions( |
39850
d89d5bc06eaa
localrepo: define "features" on repository instances (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39757
diff
changeset
|
657 repo.ui, repo.requirements, repo.features) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
658 repo._writerequirements() |