Mercurial > hg
annotate mercurial/exchangev2.py @ 40330:00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
This is somewhat hacky. For that I apologize.
At the 4.8 Sprint, we decided we wanted to land support in wireprotov2 for doing
a partial clone with changelog and manifestlog bootstrapped from a "stream clone"
like primitive.
This commit implements the client-side bits necessary to facilitate that.
If the new server-side command for obtaining raw files data is available, we
call it to get the raw files for the changelog and manifestlog. Then we
fall through to an incremental pull. But when fetching files data, instead
of using the list of a changesets and manifests that we fetched via the
"changesetdata" command, we do a linear scan of the repo and resolve the
changeset and manifest nodes along with the manifest linkrevs.
Differential Revision: https://phab.mercurial-scm.org/D5135
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Wed, 17 Oct 2018 10:10:05 +0200 |
parents | 55836a34f41b |
children | 229d23cdb203 |
rev | line source |
---|---|
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # exchangev2.py - repository exchange for wire protocol version 2 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
10 import collections |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
11 import weakref |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
12 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
13 from .i18n import _ |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 from .node import ( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 nullid, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
16 short, |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 ) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 from . import ( |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
19 bookmarks, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
20 error, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
21 mdiff, |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
22 narrowspec, |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
23 phases, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
24 pycompat, |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 setdiscovery, |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
26 ) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
27 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 def pull(pullop): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 """Pull using wire protocol version 2.""" |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 repo = pullop.repo |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 remote = pullop.remote |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
32 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
33 usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
34 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
35 # If this is a clone and it was requested to perform a "stream clone", |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
36 # we obtain the raw files data from the remote then fall back to an |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
37 # incremental pull. This is somewhat hacky and is not nearly robust enough |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
38 # for long-term usage. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
39 if usingrawchangelogandmanifest: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
40 with repo.transaction('clone'): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
41 _fetchrawstorefiles(repo, remote) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
42 repo.invalidate(clearfilecache=True) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
43 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
44 tr = pullop.trmanager.transaction() |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
45 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
46 # We don't use the repo's narrow matcher here because the patterns passed |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
47 # to exchange.pull() could be different. |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
48 narrowmatcher = narrowspec.match(repo.root, |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
49 # Empty maps to nevermatcher. So always |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
50 # set includes if missing. |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
51 pullop.includepats or {'path:.'}, |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
52 pullop.excludepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
53 |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
54 if pullop.includepats or pullop.excludepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
55 pathfilter = {} |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
56 if pullop.includepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
57 pathfilter[b'include'] = sorted(pullop.includepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
58 if pullop.excludepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
59 pathfilter[b'exclude'] = sorted(pullop.excludepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
60 else: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
61 pathfilter = None |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
62 |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
63 # Figure out what needs to be fetched. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
64 common, fetch, remoteheads = _pullchangesetdiscovery( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
65 repo, remote, pullop.heads, abortwhenunrelated=pullop.force) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
66 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
67 # And fetch the data. |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
68 pullheads = pullop.heads or remoteheads |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
69 csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
70 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
71 # New revisions are written to the changelog. But all other updates |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
72 # are deferred. Do those now. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
73 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
74 # Ensure all new changesets are draft by default. If the repo is |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
75 # publishing, the phase will be adjusted by the loop below. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
76 if csetres['added']: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
77 phases.registernew(repo, tr, phases.draft, csetres['added']) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
78 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
79 # And adjust the phase of all changesets accordingly. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
80 for phase in phases.phasenames: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
81 if phase == b'secret' or not csetres['nodesbyphase'][phase]: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
82 continue |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
83 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
84 phases.advanceboundary(repo, tr, phases.phasenames.index(phase), |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
85 csetres['nodesbyphase'][phase]) |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
86 |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
87 # Write bookmark updates. |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
88 bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'], |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
89 remote.url(), pullop.gettransaction, |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
90 explicit=pullop.explicitbookmarks) |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
91 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
92 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
93 |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
94 # If obtaining the raw store files, we need to scan the full repo to |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
95 # derive all the changesets, manifests, and linkrevs. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
96 if usingrawchangelogandmanifest: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
97 csetsforfiles = [] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
98 mnodesforfiles = [] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
99 manifestlinkrevs = {} |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
100 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
101 for rev in repo: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
102 ctx = repo[rev] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
103 mnode = ctx.manifestnode() |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
104 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
105 csetsforfiles.append(ctx.node()) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
106 mnodesforfiles.append(mnode) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
107 manifestlinkrevs[mnode] = rev |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
108 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
109 else: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
110 csetsforfiles = csetres['added'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
111 mnodesforfiles = manres['added'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
112 manifestlinkrevs = manres['linkrevs'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
113 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
114 # Find all file nodes referenced by added manifests and fetch those |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
115 # revisions. |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
116 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
117 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles, |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
118 manifestlinkrevs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
119 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
120 def _checkuserawstorefiledata(pullop): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
121 """Check whether we should use rawstorefiledata command to retrieve data.""" |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
122 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
123 repo = pullop.repo |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
124 remote = pullop.remote |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
125 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
126 # Command to obtain raw store data isn't available. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
127 if b'rawstorefiledata' not in remote.apidescriptor[b'commands']: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
128 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
129 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
130 # Only honor if user requested stream clone operation. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
131 if not pullop.streamclonerequested: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
132 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
133 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
134 # Only works on empty repos. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
135 if len(repo): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
136 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
137 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
138 # TODO This is super hacky. There needs to be a storage API for this. We |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
139 # also need to check for compatibility with the remote. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
140 if b'revlogv1' not in repo.requirements: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
141 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
142 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
143 return True |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
144 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
145 def _fetchrawstorefiles(repo, remote): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
146 with remote.commandexecutor() as e: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
147 objs = e.callcommand(b'rawstorefiledata', { |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
148 b'files': [b'changelog', b'manifestlog'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
149 }).result() |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
150 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
151 # First object is a summary of files data that follows. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
152 overall = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
153 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
154 progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
155 unit=_('bytes')) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
156 with progress: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
157 progress.update(0) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
158 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
159 # Next are pairs of file metadata, data. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
160 while True: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
161 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
162 filemeta = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
163 except StopIteration: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
164 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
165 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
166 for k in (b'location', b'path', b'size'): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
167 if k not in filemeta: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
168 raise error.Abort(_(b'remote file data missing key: %s') |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
169 % k) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
170 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
171 if filemeta[b'location'] == b'store': |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
172 vfs = repo.svfs |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
173 else: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
174 raise error.Abort(_(b'invalid location for raw file data: ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
175 b'%s') % filemeta[b'location']) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
176 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
177 bytesremaining = filemeta[b'size'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
178 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
179 with vfs.open(filemeta[b'path'], b'wb') as fh: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
180 while True: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
181 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
182 chunk = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
183 except StopIteration: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
184 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
185 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
186 bytesremaining -= len(chunk) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
187 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
188 if bytesremaining < 0: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
189 raise error.Abort(_( |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
190 b'received invalid number of bytes for file ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
191 b'data; expected %d, got extra') % |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
192 filemeta[b'size']) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
193 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
194 progress.increment(step=len(chunk)) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
195 fh.write(chunk) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
196 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
197 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
198 if chunk.islast: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
199 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
200 except AttributeError: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
201 raise error.Abort(_( |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
202 b'did not receive indefinite length bytestring ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
203 b'for file data')) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
204 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
205 if bytesremaining: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
206 raise error.Abort(_(b'received invalid number of bytes for' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
207 b'file data; expected %d got %d') % |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
208 (filemeta[b'size'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
209 filemeta[b'size'] - bytesremaining)) |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
210 |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
211 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
212 """Determine which changesets need to be pulled.""" |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
213 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
214 if heads: |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
215 knownnode = repo.changelog.hasnode |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
216 if all(knownnode(head) for head in heads): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
217 return heads, False, heads |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
218 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
219 # TODO wire protocol version 2 is capable of more efficient discovery |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
220 # than setdiscovery. Consider implementing something better. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
221 common, fetch, remoteheads = setdiscovery.findcommonheads( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
222 repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
223 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
224 common = set(common) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
225 remoteheads = set(remoteheads) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
226 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
227 # If a remote head is filtered locally, put it back in the common set. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
228 # See the comment in exchange._pulldiscoverychangegroup() for more. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
229 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
230 if fetch and remoteheads: |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
231 nodemap = repo.unfiltered().changelog.nodemap |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
232 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
233 common |= {head for head in remoteheads if head in nodemap} |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
234 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
235 if set(remoteheads).issubset(common): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
236 fetch = [] |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
237 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
238 common.discard(nullid) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
239 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
240 return common, fetch, remoteheads |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
241 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
242 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
243 # TODO consider adding a step here where we obtain the DAG shape first |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
244 # (or ask the server to slice changesets into chunks for us) so that |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
245 # we can perform multiple fetches in batches. This will facilitate |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
246 # resuming interrupted clones, higher server-side cache hit rates due |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
247 # to smaller segments, etc. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
248 with remote.commandexecutor() as e: |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
249 objs = e.callcommand(b'changesetdata', { |
40176
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
250 b'revisions': [{ |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
251 b'type': b'changesetdagrange', |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
252 b'roots': sorted(common), |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
253 b'heads': sorted(remoteheads), |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
254 }], |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
255 b'fields': {b'bookmarks', b'parents', b'phase', b'revision'}, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
256 }).result() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
257 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
258 # The context manager waits on all response data when exiting. So |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
259 # we need to remain in the context manager in order to stream data. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
260 return _processchangesetdata(repo, tr, objs) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
261 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
262 def _processchangesetdata(repo, tr, objs): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
263 repo.hook('prechangegroup', throw=True, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
264 **pycompat.strkwargs(tr.hookargs)) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
265 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
266 urepo = repo.unfiltered() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
267 cl = urepo.changelog |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
268 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
269 cl.delayupdate(tr) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
270 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
271 # The first emitted object is a header describing the data that |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
272 # follows. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
273 meta = next(objs) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
274 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
275 progress = repo.ui.makeprogress(_('changesets'), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
276 unit=_('chunks'), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
277 total=meta.get(b'totalitems')) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
278 |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
279 manifestnodes = {} |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
280 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
281 def linkrev(node): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
282 repo.ui.debug('add changeset %s\n' % short(node)) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
283 # Linkrev for changelog is always self. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
284 return len(cl) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
285 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
286 def onchangeset(cl, node): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
287 progress.increment() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
288 |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
289 revision = cl.changelogrevision(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
290 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
291 # We need to preserve the mapping of changelog revision to node |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
292 # so we can set the linkrev accordingly when manifests are added. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
293 manifestnodes[cl.rev(node)] = revision.manifest |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
294 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
295 nodesbyphase = {phase: set() for phase in phases.phasenames} |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
296 remotebookmarks = {} |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
297 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
298 # addgroup() expects a 7-tuple describing revisions. This normalizes |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
299 # the wire data to that format. |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
300 # |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
301 # This loop also aggregates non-revision metadata, such as phase |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
302 # data. |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
303 def iterrevisions(): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
304 for cset in objs: |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
305 node = cset[b'node'] |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
306 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
307 if b'phase' in cset: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
308 nodesbyphase[cset[b'phase']].add(node) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
309 |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
310 for mark in cset.get(b'bookmarks', []): |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
311 remotebookmarks[mark] = node |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
312 |
39636
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
313 # TODO add mechanism for extensions to examine records so they |
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
314 # can siphon off custom data fields. |
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
315 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
316 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
317 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
318 for field, size in cset.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
319 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
320 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
321 # Some entries might only be metadata only updates. |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
322 if b'revision' not in extrafields: |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
323 continue |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
324 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
325 data = extrafields[b'revision'] |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
326 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
327 yield ( |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
328 node, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
329 cset[b'parents'][0], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
330 cset[b'parents'][1], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
331 # Linknode is always itself for changesets. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
332 cset[b'node'], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
333 # We always send full revisions. So delta base is not set. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
334 nullid, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
335 mdiff.trivialdiffheader(len(data)) + data, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
336 # Flags not yet supported. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
337 0, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
338 ) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
339 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
340 added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
341 addrevisioncb=onchangeset) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
342 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
343 progress.complete() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
344 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
345 return { |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
346 'added': added, |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
347 'nodesbyphase': nodesbyphase, |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
348 'bookmarks': remotebookmarks, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
349 'manifestnodes': manifestnodes, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
350 } |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
351 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
352 def _fetchmanifests(repo, tr, remote, manifestnodes): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
353 rootmanifest = repo.manifestlog.getstorage(b'') |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
354 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
355 # Some manifests can be shared between changesets. Filter out revisions |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
356 # we already know about. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
357 fetchnodes = [] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
358 linkrevs = {} |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
359 seen = set() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
360 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
361 for clrev, node in sorted(manifestnodes.iteritems()): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
362 if node in seen: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
363 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
364 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
365 try: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
366 rootmanifest.rev(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
367 except error.LookupError: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
368 fetchnodes.append(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
369 linkrevs[node] = clrev |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
370 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
371 seen.add(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
372 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
373 # TODO handle tree manifests |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
374 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
375 # addgroup() expects 7-tuple describing revisions. This normalizes |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
376 # the wire data to that format. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
377 def iterrevisions(objs, progress): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
378 for manifest in objs: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
379 node = manifest[b'node'] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
380 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
381 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
382 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
383 for field, size in manifest.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
384 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
385 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
386 if b'delta' in extrafields: |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
387 basenode = manifest[b'deltabasenode'] |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
388 delta = extrafields[b'delta'] |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
389 elif b'revision' in extrafields: |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
390 basenode = nullid |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
391 revision = extrafields[b'revision'] |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
392 delta = mdiff.trivialdiffheader(len(revision)) + revision |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
393 else: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
394 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
395 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
396 yield ( |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
397 node, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
398 manifest[b'parents'][0], |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
399 manifest[b'parents'][1], |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
400 # The value passed in is passed to the lookup function passed |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
401 # to addgroup(). We already have a map of manifest node to |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
402 # changelog revision number. So we just pass in the |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
403 # manifest node here and use linkrevs.__getitem__ as the |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
404 # resolution function. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
405 node, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
406 basenode, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
407 delta, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
408 # Flags not yet supported. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
409 0 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
410 ) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
411 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
412 progress.increment() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
413 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
414 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'), |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
415 total=len(fetchnodes)) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
416 |
40173
b797150a1ab9
exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40035
diff
changeset
|
417 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata'] |
b797150a1ab9
exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40035
diff
changeset
|
418 batchsize = commandmeta.get(b'recommendedbatchsize', 10000) |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
419 # TODO make size configurable on client? |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
420 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
421 # We send commands 1 at a time to the remote. This is not the most |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
422 # efficient because we incur a round trip at the end of each batch. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
423 # However, the existing frame-based reactor keeps consuming server |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
424 # data in the background. And this results in response data buffering |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
425 # in memory. This can consume gigabytes of memory. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
426 # TODO send multiple commands in a request once background buffering |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
427 # issues are resolved. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
428 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
429 added = [] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
430 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
431 for i in pycompat.xrange(0, len(fetchnodes), batchsize): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
432 batch = [node for node in fetchnodes[i:i + batchsize]] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
433 if not batch: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
434 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
435 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
436 with remote.commandexecutor() as e: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
437 objs = e.callcommand(b'manifestdata', { |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
438 b'tree': b'', |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
439 b'nodes': batch, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
440 b'fields': {b'parents', b'revision'}, |
39641
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
441 b'haveparents': True, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
442 }).result() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
443 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
444 # Chomp off header object. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
445 next(objs) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
446 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
447 added.extend(rootmanifest.addgroup( |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
448 iterrevisions(objs, progress), |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
449 linkrevs.__getitem__, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
450 weakref.proxy(tr))) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
451 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
452 progress.complete() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
453 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
454 return { |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
455 'added': added, |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
456 'linkrevs': linkrevs, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
457 } |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
458 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
459 def _derivefilesfrommanifests(repo, matcher, manifestnodes): |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
460 """Determine what file nodes are relevant given a set of manifest nodes. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
461 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
462 Returns a dict mapping file paths to dicts of file node to first manifest |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
463 node. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
464 """ |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
465 ml = repo.manifestlog |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
466 fnodes = collections.defaultdict(dict) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
467 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
468 progress = repo.ui.makeprogress( |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
469 _('scanning manifests'), total=len(manifestnodes)) |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
470 |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
471 with progress: |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
472 for manifestnode in manifestnodes: |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
473 m = ml.get(b'', manifestnode) |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
474 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
475 # TODO this will pull in unwanted nodes because it takes the storage |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
476 # delta into consideration. What we really want is something that |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
477 # takes the delta between the manifest's parents. And ideally we |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
478 # would ignore file nodes that are known locally. For now, ignore |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
479 # both these limitations. This will result in incremental fetches |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
480 # requesting data we already have. So this is far from ideal. |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
481 md = m.readfast() |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
482 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
483 for path, fnode in md.items(): |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
484 if matcher(path): |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
485 fnodes[path].setdefault(fnode, manifestnode) |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
486 |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
487 progress.increment() |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
488 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
489 return fnodes |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
490 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
491 def _fetchfiles(repo, tr, remote, fnodes, linkrevs): |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
492 """Fetch file data from explicit file revisions.""" |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
493 def iterrevisions(objs, progress): |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
494 for filerevision in objs: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
495 node = filerevision[b'node'] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
496 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
497 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
498 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
499 for field, size in filerevision.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
500 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
501 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
502 if b'delta' in extrafields: |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
503 basenode = filerevision[b'deltabasenode'] |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
504 delta = extrafields[b'delta'] |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
505 elif b'revision' in extrafields: |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
506 basenode = nullid |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
507 revision = extrafields[b'revision'] |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
508 delta = mdiff.trivialdiffheader(len(revision)) + revision |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
509 else: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
510 continue |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
511 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
512 yield ( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
513 node, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
514 filerevision[b'parents'][0], |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
515 filerevision[b'parents'][1], |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
516 node, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
517 basenode, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
518 delta, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
519 # Flags not yet supported. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
520 0, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
521 ) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
522 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
523 progress.increment() |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
524 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
525 progress = repo.ui.makeprogress( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
526 _('files'), unit=_('chunks'), |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
527 total=sum(len(v) for v in fnodes.itervalues())) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
528 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
529 # TODO make batch size configurable |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
530 batchsize = 10000 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
531 fnodeslist = [x for x in sorted(fnodes.items())] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
532 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
533 for i in pycompat.xrange(0, len(fnodeslist), batchsize): |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
534 batch = [x for x in fnodeslist[i:i + batchsize]] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
535 if not batch: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
536 continue |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
537 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
538 with remote.commandexecutor() as e: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
539 fs = [] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
540 locallinkrevs = {} |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
541 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
542 for path, nodes in batch: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
543 fs.append((path, e.callcommand(b'filedata', { |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
544 b'path': path, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
545 b'nodes': sorted(nodes), |
39641
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
546 b'fields': {b'parents', b'revision'}, |
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
547 b'haveparents': True, |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
548 }))) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
549 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
550 locallinkrevs[path] = { |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
551 node: linkrevs[manifestnode] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
552 for node, manifestnode in nodes.iteritems()} |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
553 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
554 for path, f in fs: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
555 objs = f.result() |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
556 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
557 # Chomp off header objects. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
558 next(objs) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
559 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
560 store = repo.file(path) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
561 store.addgroup( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
562 iterrevisions(objs, progress), |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
563 locallinkrevs[path].__getitem__, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
564 weakref.proxy(tr)) |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
565 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
566 def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets, |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
567 manlinkrevs): |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
568 """Fetch file data from explicit changeset revisions.""" |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
569 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
570 def iterrevisions(objs, remaining, progress): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
571 while remaining: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
572 filerevision = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
573 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
574 node = filerevision[b'node'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
575 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
576 extrafields = {} |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
577 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
578 for field, size in filerevision.get(b'fieldsfollowing', []): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
579 extrafields[field] = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
580 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
581 if b'delta' in extrafields: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
582 basenode = filerevision[b'deltabasenode'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
583 delta = extrafields[b'delta'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
584 elif b'revision' in extrafields: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
585 basenode = nullid |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
586 revision = extrafields[b'revision'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
587 delta = mdiff.trivialdiffheader(len(revision)) + revision |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
588 else: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
589 continue |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
590 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
591 yield ( |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
592 node, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
593 filerevision[b'parents'][0], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
594 filerevision[b'parents'][1], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
595 node, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
596 basenode, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
597 delta, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
598 # Flags not yet supported. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
599 0, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
600 ) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
601 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
602 progress.increment() |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
603 remaining -= 1 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
604 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
605 progress = repo.ui.makeprogress( |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
606 _('files'), unit=_('chunks'), |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
607 total=sum(len(v) for v in fnodes.itervalues())) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
608 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
609 commandmeta = remote.apidescriptor[b'commands'][b'filesdata'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
610 batchsize = commandmeta.get(b'recommendedbatchsize', 50000) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
611 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
612 for i in pycompat.xrange(0, len(csets), batchsize): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
613 batch = [x for x in csets[i:i + batchsize]] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
614 if not batch: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
615 continue |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
616 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
617 with remote.commandexecutor() as e: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
618 args = { |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
619 b'revisions': [{ |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
620 b'type': b'changesetexplicit', |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
621 b'nodes': batch, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
622 }], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
623 b'fields': {b'parents', b'revision'}, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
624 b'haveparents': True, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
625 } |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
626 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
627 if pathfilter: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
628 args[b'pathfilter'] = pathfilter |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
629 |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
630 objs = e.callcommand(b'filesdata', args).result() |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
631 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
632 # First object is an overall header. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
633 overall = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
634 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
635 # We have overall['totalpaths'] segments. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
636 for i in pycompat.xrange(overall[b'totalpaths']): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
637 header = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
638 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
639 path = header[b'path'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
640 store = repo.file(path) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
641 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
642 linkrevs = { |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
643 fnode: manlinkrevs[mnode] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
644 for fnode, mnode in fnodes[path].iteritems()} |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
645 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
646 store.addgroup(iterrevisions(objs, header[b'totalitems'], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
647 progress), |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
648 linkrevs.__getitem__, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
649 weakref.proxy(tr)) |