Mercurial > hg
annotate mercurial/streamclone.py @ 36280:685bcdd236b5
wireprotoserver: py3 helpfully calls adds HTTP_ to CONTENT_LENGTH
Just handle both with a membership check, preferring the HTTP_
namespaced version. Sigh.
Differential Revision: https://phab.mercurial-scm.org/D2310
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 18 Feb 2018 00:02:33 -0500 |
parents | 2d3e486d09d0 |
children | 65b86ee69383 |
rev | line source |
---|---|
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - producing and consuming streaming repository data |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
10 import contextlib |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
11 import os |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
12 import struct |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
13 import tempfile |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
14 import warnings |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
15 |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
16 from .i18n import _ |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 from . import ( |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 branchmap, |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
19 cacheutil, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
20 error, |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
21 phases, |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
22 store, |
26442
ef8d27f53204
streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26441
diff
changeset
|
23 util, |
26441
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 ) |
56527b886d1d
streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
26 def canperformstreamclone(pullop, bundle2=False): |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
27 """Whether it is possible to perform a streaming clone as part of pull. |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
28 |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
29 ``bundle2`` will cause the function to consider stream clone through |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
30 bundle2 and only through bundle2. |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
31 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
32 Returns a tuple of (supported, requirements). ``supported`` is True if |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
33 streaming clone is supported and False otherwise. ``requirements`` is |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
34 a set of repo requirements from the remote, or ``None`` if stream clone |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
35 isn't supported. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
36 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
37 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
38 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
39 |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
40 bundle2supported = False |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
41 if pullop.canusebundle2: |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
42 if 'v2' in pullop.remotebundle2caps.get('stream', []): |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
43 bundle2supported = True |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
44 # else |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
45 # Server doesn't support bundle2 stream clone or doesn't support |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
46 # the versions we support. Fall back and possibly allow legacy. |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
47 |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
48 # Ensures legacy code path uses available bundle2. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
49 if bundle2supported and not bundle2: |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
50 return False, None |
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
51 # Ensures bundle2 doesn't try to do a stream clone if it isn't supported. |
35757
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
52 elif bundle2 and not bundle2supported: |
bbf7abd09ff0
streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents:
35756
diff
changeset
|
53 return False, None |
26467
ff2c89239d49
streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26466
diff
changeset
|
54 |
26447
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
55 # Streaming clone only works on empty repositories. |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
56 if len(repo): |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
57 return False, None |
591088f7028a
streamclone: add explicit check for empty local repo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26446
diff
changeset
|
58 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
59 # Streaming clone only works if all data is being requested. |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
60 if pullop.heads: |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
61 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
62 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
63 streamrequested = pullop.streamclonerequested |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
64 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
65 # If we don't have a preference, let the server decide for us. This |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
66 # likely only comes into play in LANs. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
67 if streamrequested is None: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
68 # The server can advertise whether to prefer streaming clone. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
69 streamrequested = remote.capable('stream-preferred') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
70 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
71 if not streamrequested: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
72 return False, None |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
73 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
74 # In order for stream clone to work, the client has to support all the |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
75 # requirements advertised by the server. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
76 # |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
77 # The server advertises its requirements via the "stream" and "streamreqs" |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
78 # capability. "stream" (a value-less capability) is advertised if and only |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
79 # if the only requirement is "revlogv1." Else, the "streamreqs" capability |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
80 # is advertised and contains a comma-delimited list of requirements. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
81 requirements = set() |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
82 if remote.capable('stream'): |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
83 requirements.add('revlogv1') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
84 else: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
85 streamreqs = remote.capable('streamreqs') |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
86 # This is weird and shouldn't happen with modern servers. |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
87 if not streamreqs: |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
88 pullop.repo.ui.warn(_( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
89 'warning: stream clone requested but server has them ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
90 'disabled\n')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
91 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
92 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
93 streamreqs = set(streamreqs.split(',')) |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
94 # Server requires something we don't support. Bail. |
32259
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
95 missingreqs = streamreqs - repo.supportedformats |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
96 if missingreqs: |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
97 pullop.repo.ui.warn(_( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
98 'warning: stream clone requested but client is missing ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
99 'requirements: %s\n') % ', '.join(sorted(missingreqs))) |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
100 pullop.repo.ui.warn( |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
101 _('(see https://www.mercurial-scm.org/wiki/MissingRequirement ' |
076f1ff43f0f
clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents:
30975
diff
changeset
|
102 'for more information)\n')) |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
103 return False, None |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
104 requirements = streamreqs |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
105 |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
106 return True, requirements |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
107 |
26462
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
108 def maybeperformlegacystreamclone(pullop): |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
109 """Possibly perform a legacy stream clone operation. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
110 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
111 Legacy stream clones are performed as part of pull but before all other |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
112 operations. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
113 |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
114 A legacy stream clone will not be performed if a bundle2 stream clone is |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
115 supported. |
3b0ec09192ae
streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26461
diff
changeset
|
116 """ |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
117 supported, requirements = canperformstreamclone(pullop) |
26458
362793295640
streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26447
diff
changeset
|
118 |
26446
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
119 if not supported: |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
120 return |
3ea10bb761ce
streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26445
diff
changeset
|
121 |
26466
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
122 repo = pullop.repo |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
123 remote = pullop.remote |
3515db5aae05
streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26462
diff
changeset
|
124 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
125 # Save remote branchmap. We will use it later to speed up branchcache |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
126 # creation. |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
127 rbranchmap = None |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
128 if remote.capable('branchmap'): |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
129 rbranchmap = remote.branchmap() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
130 |
26470
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
131 repo.ui.status(_('streaming all changes\n')) |
4b5647d9ee13
streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26469
diff
changeset
|
132 |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
133 fp = remote.stream_out() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
134 l = fp.readline() |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
135 try: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
136 resp = int(l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
137 except ValueError: |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
138 raise error.ResponseError( |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
139 _('unexpected response from remote server:'), l) |
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
140 if resp == 1: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
141 raise error.Abort(_('operation forbidden by server')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
142 elif resp == 2: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
143 raise error.Abort(_('locking the remote repository failed')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
144 elif resp != 0: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26470
diff
changeset
|
145 raise error.Abort(_('the server sent an unknown error code')) |
26459
3b28ffde133a
streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26458
diff
changeset
|
146 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
147 l = fp.readline() |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
148 try: |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
149 filecount, bytecount = map(int, l.split(' ', 1)) |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
150 except (ValueError, TypeError): |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
151 raise error.ResponseError( |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
152 _('unexpected response from remote server:'), l) |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
153 |
27850
49cfddbf54ba
with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents:
27845
diff
changeset
|
154 with repo.lock(): |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
155 consumev1(repo, fp, filecount, bytecount) |
26461
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
156 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
157 # new requirements = old non-format requirements + |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
158 # new format-related remote requirements |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
159 # requirements from the streamed-in repository |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
160 repo.requirements = requirements | ( |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
161 repo.requirements - repo.supportedformats) |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
162 repo._applyopenerreqs() |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
163 repo._writerequirements() |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
164 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
165 if rbranchmap: |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
166 branchmap.replacecache(repo, rbranchmap) |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
167 |
09cc3c2e9ece
streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26460
diff
changeset
|
168 repo.invalidate() |
26445
f134fb33c906
streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26444
diff
changeset
|
169 |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
170 def allowservergeneration(repo): |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
171 """Whether streaming clones are allowed from the server.""" |
33499
0407a51b9d8c
codemod: register core configitems using a script
Jun Wu <quark@fb.com>
parents:
33411
diff
changeset
|
172 if not repo.ui.configbool('server', 'uncompressed', untrusted=True): |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
173 return False |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
174 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
175 # The way stream clone works makes it impossible to hide secret changesets. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
176 # So don't allow this by default. |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
177 secret = phases.hassecret(repo) |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
178 if secret: |
33226
b045344fe35e
configitems: register the 'server.uncompressedallowsecret' config
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
32744
diff
changeset
|
179 return repo.ui.configbool('server', 'uncompressedallowsecret') |
32744
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
180 |
33b7283a3828
streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32259
diff
changeset
|
181 return True |
26444
623743010133
streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26443
diff
changeset
|
182 |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
183 # This is it's own function so extensions can override it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
184 def _walkstreamfiles(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
185 return repo.store.walk() |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
186 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
187 def generatev1(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
188 """Emit content for version 1 of a streaming clone. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
189 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
190 This returns a 3-tuple of (file count, byte size, data iterator). |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
191 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
192 The data iterator consists of N entries for each file being transferred. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
193 Each file entry starts as a line with the file name and integer size |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
194 delimited by a null byte. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
195 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
196 The raw file data follows. Following the raw file data is the next file |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
197 entry, or EOF. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
198 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
199 When used on the wire protocol, an additional line indicating protocol |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
200 success will be prepended to the stream. This function is not responsible |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
201 for adding it. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
202 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
203 This function will obtain a repository lock to ensure a consistent view of |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
204 the store is captured. It therefore may raise LockError. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
205 """ |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
206 entries = [] |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
207 total_bytes = 0 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
208 # Get consistent snapshot of repo, lock during scan. |
27845
7417e1c10253
with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27794
diff
changeset
|
209 with repo.lock(): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
210 repo.ui.debug('scanning\n') |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
211 for name, ename, size in _walkstreamfiles(repo): |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
212 if size: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
213 entries.append((name, size)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
214 total_bytes += size |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
215 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
216 repo.ui.debug('%d files, %d bytes to transfer\n' % |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
217 (len(entries), total_bytes)) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
218 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
219 svfs = repo.svfs |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
220 debugflag = repo.ui.debugflag |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
221 |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
222 def emitrevlogdata(): |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
223 for name, size in entries: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
224 if debugflag: |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
225 repo.ui.debug('sending %s (%d bytes)\n' % (name, size)) |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
226 # partially encode name over the wire for backwards compat |
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
227 yield '%s\0%d\n' % (store.encodedir(name), size) |
33411
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
228 # auditing at this stage is both pointless (paths are already |
50b49bb0fff3
streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33410
diff
changeset
|
229 # trusted by the local repo) and expensive |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
230 with svfs(name, 'rb', auditpath=False) as fp: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
231 if size <= 65536: |
33258
761ccfeff8b1
streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
33257
diff
changeset
|
232 yield fp.read(size) |
33410
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
233 else: |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
234 for chunk in util.filechunkiter(fp, limit=size): |
c784308305c6
streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents:
33258
diff
changeset
|
235 yield chunk |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
236 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
237 return len(entries), total_bytes, emitrevlogdata() |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
238 |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
239 def generatev1wireproto(repo): |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
240 """Emit content for version 1 of streaming clone suitable for the wire. |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
241 |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
242 This is the data output from ``generatev1()`` with 2 header lines. The |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
243 first line indicates overall success. The 2nd contains the file count and |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
244 byte size of payload. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
245 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
246 The success line contains "0" for success, "1" for stream generation not |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
247 allowed, and "2" for error locking the repository (possibly indicating |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
248 a permissions error for the server process). |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
249 """ |
35491
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
250 if not allowservergeneration(repo): |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
251 yield '1\n' |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
252 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
253 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
254 try: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
255 filecount, bytecount, it = generatev1(repo) |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
256 except error.LockError: |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
257 yield '2\n' |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
258 return |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
259 |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
260 # Indicates successful response. |
ded3a63f305b
streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents:
33499
diff
changeset
|
261 yield '0\n' |
26469
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
262 yield '%d %d\n' % (filecount, bytecount) |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
263 for chunk in it: |
fb743268510e
streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26468
diff
changeset
|
264 yield chunk |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
265 |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
266 def generatebundlev1(repo, compression='UN'): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
267 """Emit content for version 1 of a stream clone bundle. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
268 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
269 The first 4 bytes of the output ("HGS1") denote this as stream clone |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
270 bundle version 1. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
271 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
272 The next 2 bytes indicate the compression type. Only "UN" is currently |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
273 supported. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
274 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
275 The next 16 bytes are two 64-bit big endian unsigned integers indicating |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
276 file count and byte count, respectively. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
277 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
278 The next 2 bytes is a 16-bit big endian unsigned short declaring the length |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
279 of the requirements string, including a trailing \0. The following N bytes |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
280 are the requirements string, which is ASCII containing a comma-delimited |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
281 list of repo requirements that are needed to support the data. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
282 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
283 The remaining content is the output of ``generatev1()`` (which may be |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
284 compressed in the future). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
285 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
286 Returns a tuple of (requirements, data generator). |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
287 """ |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
288 if compression != 'UN': |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
289 raise ValueError('we do not support the compression argument yet') |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
290 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
291 requirements = repo.requirements & repo.supportedformats |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
292 requires = ','.join(sorted(requirements)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
293 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
294 def gen(): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
295 yield 'HGS1' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
296 yield compression |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
297 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
298 filecount, bytecount, it = generatev1(repo) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
299 repo.ui.status(_('writing %d bytes for %d files\n') % |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
300 (bytecount, filecount)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
301 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
302 yield struct.pack('>QQ', filecount, bytecount) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
303 yield struct.pack('>H', len(requires) + 1) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
304 yield requires + '\0' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
305 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
306 # This is where we'll add compression in the future. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
307 assert compression == 'UN' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
308 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
309 seen = 0 |
28473
61d1a3cc6e1c
streamclone: specify unit for ui.progress when handling data
Anton Shestakov <av6@dwimlabs.net>
parents:
27897
diff
changeset
|
310 repo.ui.progress(_('bundle'), 0, total=bytecount, unit=_('bytes')) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
311 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
312 for chunk in it: |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
313 seen += len(chunk) |
28473
61d1a3cc6e1c
streamclone: specify unit for ui.progress when handling data
Anton Shestakov <av6@dwimlabs.net>
parents:
27897
diff
changeset
|
314 repo.ui.progress(_('bundle'), seen, total=bytecount, |
61d1a3cc6e1c
streamclone: specify unit for ui.progress when handling data
Anton Shestakov <av6@dwimlabs.net>
parents:
27897
diff
changeset
|
315 unit=_('bytes')) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
316 yield chunk |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
317 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
318 repo.ui.progress(_('bundle'), None) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
319 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
320 return requirements, gen() |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
321 |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
322 def consumev1(repo, fp, filecount, bytecount): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
323 """Apply the contents from version 1 of a streaming clone file handle. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
324 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
325 This takes the output from "stream_out" and applies it to the specified |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
326 repository. |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
327 |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
328 Like "stream_out," the status line added by the wire protocol is not |
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
29919
diff
changeset
|
329 handled by this function. |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
330 """ |
27859
f55a5ace8e69
with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27850
diff
changeset
|
331 with repo.lock(): |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
332 repo.ui.status(_('%d files to transfer, %s of data\n') % |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
333 (filecount, util.bytecount(bytecount))) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
334 handled_bytes = 0 |
28473
61d1a3cc6e1c
streamclone: specify unit for ui.progress when handling data
Anton Shestakov <av6@dwimlabs.net>
parents:
27897
diff
changeset
|
335 repo.ui.progress(_('clone'), 0, total=bytecount, unit=_('bytes')) |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
336 start = util.timer() |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
337 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
338 # TODO: get rid of (potential) inconsistency |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
339 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
340 # If transaction is started and any @filecache property is |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
341 # changed at this point, it causes inconsistency between |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
342 # in-memory cached property and streamclone-ed file on the |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
343 # disk. Nested transaction prevents transaction scope "clone" |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
344 # below from writing in-memory changes out at the end of it, |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
345 # even though in-memory changes are discarded at the end of it |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
346 # regardless of transaction nesting. |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
347 # |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
348 # But transaction nesting can't be simply prohibited, because |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
349 # nesting occurs also in ordinary case (e.g. enabling |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
350 # clonebundles). |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
351 |
27870
ed9950ba091e
with: use context manager for transaction in consumev1
Bryan O'Sullivan <bryano@fb.com>
parents:
27859
diff
changeset
|
352 with repo.transaction('clone'): |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
353 with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
354 for i in xrange(filecount): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
355 # XXX doesn't support '\n' or '\r' in filenames |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
356 l = fp.readline() |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
357 try: |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
358 name, size = l.split('\0', 1) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
359 size = int(size) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
360 except (ValueError, TypeError): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
361 raise error.ResponseError( |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
362 _('unexpected response from remote server:'), l) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
363 if repo.ui.debugflag: |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
364 repo.ui.debug('adding %s (%s)\n' % |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
365 (name, util.bytecount(size))) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
366 # for backwards compat, name was partially encoded |
27897
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
367 path = store.decodedir(name) |
2fdbf22a1b63
streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27896
diff
changeset
|
368 with repo.svfs(path, 'w', backgroundclose=True) as ofp: |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
369 for chunk in util.filechunkiter(fp, limit=size): |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
370 handled_bytes += len(chunk) |
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
371 repo.ui.progress(_('clone'), handled_bytes, |
28473
61d1a3cc6e1c
streamclone: specify unit for ui.progress when handling data
Anton Shestakov <av6@dwimlabs.net>
parents:
27897
diff
changeset
|
372 total=bytecount, unit=_('bytes')) |
27896
1d29893240cc
streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27882
diff
changeset
|
373 ofp.write(chunk) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
374 |
29919
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
375 # force @filecache properties to be reloaded from |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
376 # streamclone-ed file at next access |
519a02267f90
streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29917
diff
changeset
|
377 repo.invalidate(clearfilecache=True) |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
378 |
30975
22fbca1d11ed
mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents:
30332
diff
changeset
|
379 elapsed = util.timer() - start |
26443
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
380 if elapsed <= 0: |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
381 elapsed = 0.001 |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
382 repo.ui.progress(_('clone'), None) |
d947086d8973
streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26442
diff
changeset
|
383 repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % |
26468
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
384 (util.bytecount(bytecount), elapsed, |
19bbd53af46d
streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26467
diff
changeset
|
385 util.bytecount(bytecount / elapsed))) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
386 |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
387 def readbundle1header(fp): |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
388 compression = fp.read(2) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
389 if compression != 'UN': |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
390 raise error.Abort(_('only uncompressed stream clone bundles are ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
391 'supported; got %s') % compression) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
392 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
393 filecount, bytecount = struct.unpack('>QQ', fp.read(16)) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
394 requireslen = struct.unpack('>H', fp.read(2))[0] |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
395 requires = fp.read(requireslen) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
396 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
397 if not requires.endswith('\0'): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
398 raise error.Abort(_('malformed stream clone bundle: ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
399 'requirements not properly encoded')) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
400 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
401 requirements = set(requires.rstrip('\0').split(',')) |
27882
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
402 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
403 return filecount, bytecount, requirements |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
404 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
405 def applybundlev1(repo, fp): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
406 """Apply the content from a stream clone bundle version 1. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
407 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
408 We assume the 4 byte header has been read and validated and the file handle |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
409 is at the 2 byte compression identifier. |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
410 """ |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
411 if len(repo): |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
412 raise error.Abort(_('cannot apply stream clone bundle on non-empty ' |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
413 'repo')) |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
414 |
319b0bf6ecc9
streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27870
diff
changeset
|
415 filecount, bytecount, requirements = readbundle1header(fp) |
26755
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
416 missingreqs = requirements - repo.supportedformats |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
417 if missingreqs: |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
418 raise error.Abort(_('unable to apply stream clone: ' |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
419 'unsupported format: %s') % |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
420 ', '.join(sorted(missingreqs))) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
421 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
422 consumev1(repo, fp, filecount, bytecount) |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
423 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
424 class streamcloneapplier(object): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
425 """Class to manage applying streaming clone bundles. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
426 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
427 We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
428 readers to perform bundle type-specific functionality. |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
429 """ |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
430 def __init__(self, fh): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
431 self._fh = fh |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
432 |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
433 def apply(self, repo): |
bb0b955d050d
streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
434 return applybundlev1(repo, self._fh) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
435 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
436 # type of file to stream |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
437 _fileappend = 0 # append only file |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
438 _filefull = 1 # full snapshot file |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
439 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
440 # Source of the file |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
441 _srcstore = 's' # store (svfs) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
442 _srccache = 'c' # cache (cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
443 |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
444 # This is it's own function so extensions can override it. |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
445 def _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
446 """list snapshot file from the store""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
447 fnames = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
448 if not repo.publishing(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
449 fnames.append('phaseroots') |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
450 return fnames |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
451 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
452 def _filterfull(entry, copy, vfsmap): |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
453 """actually copy the snapshot files""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
454 src, name, ftype, data = entry |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
455 if ftype != _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
456 return entry |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
457 return (src, name, ftype, copy(vfsmap[src].join(name))) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
458 |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
459 @contextlib.contextmanager |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
460 def maketempcopies(): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
461 """return a function to temporary copy file""" |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
462 files = [] |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
463 try: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
464 def copy(src): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
465 fd, dst = tempfile.mkstemp() |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
466 os.close(fd) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
467 files.append(dst) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
468 util.copyfiles(src, dst, hardlink=True) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
469 return dst |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
470 yield copy |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
471 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
472 for tmp in files: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
473 util.tryunlink(tmp) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
474 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
475 def _makemap(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
476 """make a (src -> vfs) map for the repo""" |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
477 vfsmap = { |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
478 _srcstore: repo.svfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
479 _srccache: repo.cachevfs, |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
480 } |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
481 # we keep repo.vfs out of the on purpose, ther are too many danger there |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
482 # (eg: .hg/hgrc) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
483 assert repo.vfs not in vfsmap.values() |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
484 |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
485 return vfsmap |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
486 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
487 def _emit2(repo, entries, totalfilesize): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
488 """actually emit the stream bundle""" |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
489 vfsmap = _makemap(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
490 progress = repo.ui.progress |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
491 progress(_('bundle'), 0, total=totalfilesize, unit=_('bytes')) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
492 with maketempcopies() as copy: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
493 try: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
494 # copy is delayed until we are in the try |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
495 entries = [_filterfull(e, copy, vfsmap) for e in entries] |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
496 yield None # this release the lock on the repository |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
497 seen = 0 |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
498 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
499 for src, name, ftype, data in entries: |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
500 vfs = vfsmap[src] |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
501 yield src |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
502 yield util.uvarintencode(len(name)) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
503 if ftype == _fileappend: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
504 fp = vfs(name) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
505 size = data |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
506 elif ftype == _filefull: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
507 fp = open(data, 'rb') |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
508 size = util.fstat(fp).st_size |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
509 try: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
510 yield util.uvarintencode(size) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
511 yield name |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
512 if size <= 65536: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
513 chunks = (fp.read(size),) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
514 else: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
515 chunks = util.filechunkiter(fp, limit=size) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
516 for chunk in chunks: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
517 seen += len(chunk) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
518 progress(_('bundle'), seen, total=totalfilesize, |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
519 unit=_('bytes')) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
520 yield chunk |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
521 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
522 fp.close() |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
523 finally: |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
524 progress(_('bundle'), None) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
525 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
526 def generatev2(repo): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
527 """Emit content for version 2 of a streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
528 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
529 the data stream consists the following entries: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
530 1) A char representing the file destination (eg: store or cache) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
531 2) A varint containing the length of the filename |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
532 3) A varint containing the length of file data |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
533 4) N bytes containing the filename (the internal, store-agnostic form) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
534 5) N bytes containing the file data |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
535 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
536 Returns a 3-tuple of (file count, file size, data iterator). |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
537 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
538 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
539 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
540 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
541 entries = [] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
542 totalfilesize = 0 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
543 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
544 repo.ui.debug('scanning\n') |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
545 for name, ename, size in _walkstreamfiles(repo): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
546 if size: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
547 entries.append((_srcstore, name, _fileappend, size)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
548 totalfilesize += size |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
549 for name in _walkstreamfullstorefiles(repo): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
550 if repo.svfs.exists(name): |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
551 totalfilesize += repo.svfs.lstat(name).st_size |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
552 entries.append((_srcstore, name, _filefull, None)) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
553 for name in cacheutil.cachetocopy(repo): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
554 if repo.cachevfs.exists(name): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
555 totalfilesize += repo.cachevfs.lstat(name).st_size |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
556 entries.append((_srccache, name, _filefull, None)) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
557 |
35802
bbc07357b567
streamclone: rename '_emit' to '_emit2' for clarity
Boris Feld <boris.feld@octobus.net>
parents:
35767
diff
changeset
|
558 chunks = _emit2(repo, entries, totalfilesize) |
35765
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
559 first = next(chunks) |
56c30b31afbe
streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents:
35757
diff
changeset
|
560 assert first is None |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
561 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
562 return len(entries), totalfilesize, chunks |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
563 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
564 @contextlib.contextmanager |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
565 def nested(*ctxs): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
566 with warnings.catch_warnings(): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
567 # For some reason, Python decided 'nested' was deprecated without |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
568 # replacement. They officially advertised for filtering the deprecation |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
569 # warning for people who actually need the feature. |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
570 warnings.filterwarnings("ignore",category=DeprecationWarning) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
571 with contextlib.nested(*ctxs): |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
572 yield |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
573 |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
574 def consumev2(repo, fp, filecount, filesize): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
575 """Apply the contents from a version 2 streaming clone. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
576 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
577 Data is read from an object that only needs to provide a ``read(size)`` |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
578 method. |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
579 """ |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
580 with repo.lock(): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
581 repo.ui.status(_('%d files to transfer, %s of data\n') % |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
582 (filecount, util.bytecount(filesize))) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
583 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
584 start = util.timer() |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
585 handledbytes = 0 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
586 progress = repo.ui.progress |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
587 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
588 progress(_('clone'), handledbytes, total=filesize, unit=_('bytes')) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
589 |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
590 vfsmap = _makemap(repo) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
591 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
592 with repo.transaction('clone'): |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
593 ctxs = (vfs.backgroundclosing(repo.ui) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
594 for vfs in vfsmap.values()) |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
595 with nested(*ctxs): |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
596 for i in range(filecount): |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
597 src = util.readexactly(fp, 1) |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
598 vfs = vfsmap[src] |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
599 namelen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
600 datalen = util.uvarintdecodestream(fp) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
601 |
35803
3ad3aaeb1134
streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents:
35802
diff
changeset
|
602 name = util.readexactly(fp, namelen) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
603 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
604 if repo.ui.debugflag: |
35767
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
605 repo.ui.debug('adding [%s] %s (%s)\n' % |
5f5fb279fd39
streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents:
35765
diff
changeset
|
606 (src, name, util.bytecount(datalen))) |
35756
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
607 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
608 with vfs(name, 'w') as ofp: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
609 for chunk in util.filechunkiter(fp, limit=datalen): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
610 handledbytes += len(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
611 progress(_('clone'), handledbytes, total=filesize, |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
612 unit=_('bytes')) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
613 ofp.write(chunk) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
614 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
615 # force @filecache properties to be reloaded from |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
616 # streamclone-ed file at next access |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
617 repo.invalidate(clearfilecache=True) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
618 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
619 elapsed = util.timer() - start |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
620 if elapsed <= 0: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
621 elapsed = 0.001 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
622 progress(_('clone'), None) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
623 repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
624 (util.bytecount(handledbytes), elapsed, |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
625 util.bytecount(handledbytes / elapsed))) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
626 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
627 def applybundlev2(repo, fp, filecount, filesize, requirements): |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
628 missingreqs = [r for r in requirements if r not in repo.supported] |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
629 if missingreqs: |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
630 raise error.Abort(_('unable to apply stream clone: ' |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
631 'unsupported format: %s') % |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
632 ', '.join(sorted(missingreqs))) |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
633 |
cfdccd560b66
streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents:
35491
diff
changeset
|
634 consumev2(repo, fp, filecount, filesize) |
35804
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
635 |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
636 # new requirements = old non-format requirements + |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
637 # new format-related remote requirements |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
638 # requirements from the streamed-in repository |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
639 repo.requirements = set(requirements) | ( |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
640 repo.requirements - repo.supportedformats) |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
641 repo._applyopenerreqs() |
2d3e486d09d0
streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents:
35803
diff
changeset
|
642 repo._writerequirements() |