Mercurial > hg
annotate mercurial/exchangev2.py @ 42924:e0bf41b83cef
exchange: avoid unnecessary conversion of bookmark nodes to hex (API)
Differential Revision: https://phab.mercurial-scm.org/D6830
author | Valentin Gatien-Baron <valentin.gatienbaron@gmail.com> |
---|---|
date | Sun, 08 Sep 2019 20:10:32 -0400 |
parents | 268662aac075 |
children | 2372284d9457 |
rev | line source |
---|---|
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # exchangev2.py - repository exchange for wire protocol version 2 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
10 import collections |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
11 import weakref |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
12 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
13 from .i18n import _ |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 from .node import ( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 nullid, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
16 short, |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 ) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 from . import ( |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
19 bookmarks, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
20 error, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
21 mdiff, |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
22 narrowspec, |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
23 phases, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
24 pycompat, |
42813
268662aac075
interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41217
diff
changeset
|
25 setdiscovery, |
268662aac075
interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41217
diff
changeset
|
26 ) |
268662aac075
interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41217
diff
changeset
|
27 from .interfaces import ( |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
28 repository, |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 ) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 def pull(pullop): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 """Pull using wire protocol version 2.""" |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 repo = pullop.repo |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
34 remote = pullop.remote |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
35 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
36 usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
37 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
38 # If this is a clone and it was requested to perform a "stream clone", |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
39 # we obtain the raw files data from the remote then fall back to an |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
40 # incremental pull. This is somewhat hacky and is not nearly robust enough |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
41 # for long-term usage. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
42 if usingrawchangelogandmanifest: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
43 with repo.transaction('clone'): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
44 _fetchrawstorefiles(repo, remote) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
45 repo.invalidate(clearfilecache=True) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
46 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
47 tr = pullop.trmanager.transaction() |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
48 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
49 # We don't use the repo's narrow matcher here because the patterns passed |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
50 # to exchange.pull() could be different. |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
51 narrowmatcher = narrowspec.match(repo.root, |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
52 # Empty maps to nevermatcher. So always |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
53 # set includes if missing. |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
54 pullop.includepats or {'path:.'}, |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
55 pullop.excludepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
56 |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
57 if pullop.includepats or pullop.excludepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
58 pathfilter = {} |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
59 if pullop.includepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
60 pathfilter[b'include'] = sorted(pullop.includepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
61 if pullop.excludepats: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
62 pathfilter[b'exclude'] = sorted(pullop.excludepats) |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
63 else: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
64 pathfilter = None |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
65 |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
66 # Figure out what needs to be fetched. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
67 common, fetch, remoteheads = _pullchangesetdiscovery( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
68 repo, remote, pullop.heads, abortwhenunrelated=pullop.force) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
69 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
70 # And fetch the data. |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
71 pullheads = pullop.heads or remoteheads |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
72 csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
73 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
74 # New revisions are written to the changelog. But all other updates |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
75 # are deferred. Do those now. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
76 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
77 # Ensure all new changesets are draft by default. If the repo is |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
78 # publishing, the phase will be adjusted by the loop below. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
79 if csetres['added']: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
80 phases.registernew(repo, tr, phases.draft, csetres['added']) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
81 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
82 # And adjust the phase of all changesets accordingly. |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
83 for phase in phases.phasenames: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
84 if phase == b'secret' or not csetres['nodesbyphase'][phase]: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
85 continue |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
86 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
87 phases.advanceboundary(repo, tr, phases.phasenames.index(phase), |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
88 csetres['nodesbyphase'][phase]) |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
89 |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
90 # Write bookmark updates. |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
91 bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'], |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
92 remote.url(), pullop.gettransaction, |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
93 explicit=pullop.explicitbookmarks) |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
94 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
95 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
96 |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
97 # We don't properly support shallow changeset and manifest yet. So we apply |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
98 # depth limiting locally. |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
99 if pullop.depth: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
100 relevantcsetnodes = set() |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
101 clnode = repo.changelog.node |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
102 |
41217
afa884015e66
revset: remove the last usage of "%d" for a non-revision entry
Boris Feld <boris.feld@octobus.net>
parents:
40393
diff
changeset
|
103 for rev in repo.revs(b'ancestors(%ln, %s)', |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
104 pullheads, pullop.depth - 1): |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
105 relevantcsetnodes.add(clnode(rev)) |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
106 |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
107 csetrelevantfilter = lambda n: n in relevantcsetnodes |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
108 |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
109 else: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
110 csetrelevantfilter = lambda n: True |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
111 |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
112 # If obtaining the raw store files, we need to scan the full repo to |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
113 # derive all the changesets, manifests, and linkrevs. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
114 if usingrawchangelogandmanifest: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
115 csetsforfiles = [] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
116 mnodesforfiles = [] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
117 manifestlinkrevs = {} |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
118 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
119 for rev in repo: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
120 ctx = repo[rev] |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
121 node = ctx.node() |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
122 |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
123 if not csetrelevantfilter(node): |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
124 continue |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
125 |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
126 mnode = ctx.manifestnode() |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
127 |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
128 csetsforfiles.append(node) |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
129 mnodesforfiles.append(mnode) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
130 manifestlinkrevs[mnode] = rev |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
131 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
132 else: |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
133 csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)] |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
134 mnodesforfiles = manres['added'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
135 manifestlinkrevs = manres['linkrevs'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
136 |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
137 # Find all file nodes referenced by added manifests and fetch those |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
138 # revisions. |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
139 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
140 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles, |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
141 manifestlinkrevs, shallow=bool(pullop.depth)) |
40330
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
142 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
143 def _checkuserawstorefiledata(pullop): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
144 """Check whether we should use rawstorefiledata command to retrieve data.""" |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
145 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
146 repo = pullop.repo |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
147 remote = pullop.remote |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
148 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
149 # Command to obtain raw store data isn't available. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
150 if b'rawstorefiledata' not in remote.apidescriptor[b'commands']: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
151 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
152 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
153 # Only honor if user requested stream clone operation. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
154 if not pullop.streamclonerequested: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
155 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
156 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
157 # Only works on empty repos. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
158 if len(repo): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
159 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
160 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
161 # TODO This is super hacky. There needs to be a storage API for this. We |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
162 # also need to check for compatibility with the remote. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
163 if b'revlogv1' not in repo.requirements: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
164 return False |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
165 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
166 return True |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
167 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
168 def _fetchrawstorefiles(repo, remote): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
169 with remote.commandexecutor() as e: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
170 objs = e.callcommand(b'rawstorefiledata', { |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
171 b'files': [b'changelog', b'manifestlog'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
172 }).result() |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
173 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
174 # First object is a summary of files data that follows. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
175 overall = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
176 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
177 progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
178 unit=_('bytes')) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
179 with progress: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
180 progress.update(0) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
181 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
182 # Next are pairs of file metadata, data. |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
183 while True: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
184 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
185 filemeta = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
186 except StopIteration: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
187 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
188 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
189 for k in (b'location', b'path', b'size'): |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
190 if k not in filemeta: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
191 raise error.Abort(_(b'remote file data missing key: %s') |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
192 % k) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
193 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
194 if filemeta[b'location'] == b'store': |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
195 vfs = repo.svfs |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
196 else: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
197 raise error.Abort(_(b'invalid location for raw file data: ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
198 b'%s') % filemeta[b'location']) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
199 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
200 bytesremaining = filemeta[b'size'] |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
201 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
202 with vfs.open(filemeta[b'path'], b'wb') as fh: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
203 while True: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
204 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
205 chunk = next(objs) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
206 except StopIteration: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
207 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
208 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
209 bytesremaining -= len(chunk) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
210 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
211 if bytesremaining < 0: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
212 raise error.Abort(_( |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
213 b'received invalid number of bytes for file ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
214 b'data; expected %d, got extra') % |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
215 filemeta[b'size']) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
216 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
217 progress.increment(step=len(chunk)) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
218 fh.write(chunk) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
219 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
220 try: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
221 if chunk.islast: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
222 break |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
223 except AttributeError: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
224 raise error.Abort(_( |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
225 b'did not receive indefinite length bytestring ' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
226 b'for file data')) |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
227 |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
228 if bytesremaining: |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
229 raise error.Abort(_(b'received invalid number of bytes for' |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
230 b'file data; expected %d got %d') % |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
231 (filemeta[b'size'], |
00a4cd368e3f
exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40327
diff
changeset
|
232 filemeta[b'size'] - bytesremaining)) |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
233 |
39629
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
234 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
235 """Determine which changesets need to be pulled.""" |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
236 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
237 if heads: |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
238 knownnode = repo.changelog.hasnode |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
239 if all(knownnode(head) for head in heads): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
240 return heads, False, heads |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
241 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
242 # TODO wire protocol version 2 is capable of more efficient discovery |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
243 # than setdiscovery. Consider implementing something better. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
244 common, fetch, remoteheads = setdiscovery.findcommonheads( |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
245 repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
246 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
247 common = set(common) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
248 remoteheads = set(remoteheads) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
249 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
250 # If a remote head is filtered locally, put it back in the common set. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
251 # See the comment in exchange._pulldiscoverychangegroup() for more. |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
252 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
253 if fetch and remoteheads: |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
254 nodemap = repo.unfiltered().changelog.nodemap |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
255 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
256 common |= {head for head in remoteheads if head in nodemap} |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
257 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
258 if set(remoteheads).issubset(common): |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
259 fetch = [] |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
260 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
261 common.discard(nullid) |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
262 |
a86d21e70b2b
exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
263 return common, fetch, remoteheads |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
264 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
265 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
266 # TODO consider adding a step here where we obtain the DAG shape first |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
267 # (or ask the server to slice changesets into chunks for us) so that |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
268 # we can perform multiple fetches in batches. This will facilitate |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
269 # resuming interrupted clones, higher server-side cache hit rates due |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
270 # to smaller segments, etc. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
271 with remote.commandexecutor() as e: |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
272 objs = e.callcommand(b'changesetdata', { |
40176
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
273 b'revisions': [{ |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
274 b'type': b'changesetdagrange', |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
275 b'roots': sorted(common), |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
276 b'heads': sorted(remoteheads), |
41263df08109
wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40173
diff
changeset
|
277 }], |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
278 b'fields': {b'bookmarks', b'parents', b'phase', b'revision'}, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
279 }).result() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
280 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
281 # The context manager waits on all response data when exiting. So |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
282 # we need to remain in the context manager in order to stream data. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
283 return _processchangesetdata(repo, tr, objs) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
284 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
285 def _processchangesetdata(repo, tr, objs): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
286 repo.hook('prechangegroup', throw=True, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
287 **pycompat.strkwargs(tr.hookargs)) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
288 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
289 urepo = repo.unfiltered() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
290 cl = urepo.changelog |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
291 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
292 cl.delayupdate(tr) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
293 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
294 # The first emitted object is a header describing the data that |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
295 # follows. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
296 meta = next(objs) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
297 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
298 progress = repo.ui.makeprogress(_('changesets'), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
299 unit=_('chunks'), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
300 total=meta.get(b'totalitems')) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
301 |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
302 manifestnodes = {} |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
303 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
304 def linkrev(node): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
305 repo.ui.debug('add changeset %s\n' % short(node)) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
306 # Linkrev for changelog is always self. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
307 return len(cl) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
308 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
309 def onchangeset(cl, node): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
310 progress.increment() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
311 |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
312 revision = cl.changelogrevision(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
313 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
314 # We need to preserve the mapping of changelog revision to node |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
315 # so we can set the linkrev accordingly when manifests are added. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
316 manifestnodes[cl.rev(node)] = revision.manifest |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
317 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
318 nodesbyphase = {phase: set() for phase in phases.phasenames} |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
319 remotebookmarks = {} |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
320 |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
321 # addgroup() expects a 7-tuple describing revisions. This normalizes |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
322 # the wire data to that format. |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
323 # |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
324 # This loop also aggregates non-revision metadata, such as phase |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
325 # data. |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
326 def iterrevisions(): |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
327 for cset in objs: |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
328 node = cset[b'node'] |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
329 |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
330 if b'phase' in cset: |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
331 nodesbyphase[cset[b'phase']].add(node) |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
332 |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
333 for mark in cset.get(b'bookmarks', []): |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
334 remotebookmarks[mark] = node |
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
335 |
39636
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
336 # TODO add mechanism for extensions to examine records so they |
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
337 # can siphon off custom data fields. |
399ddd3227a4
wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39635
diff
changeset
|
338 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
339 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
340 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
341 for field, size in cset.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
342 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
343 |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
344 # Some entries might only be metadata only updates. |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
345 if b'revision' not in extrafields: |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
346 continue |
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
347 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
348 data = extrafields[b'revision'] |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
349 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
350 yield ( |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
351 node, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
352 cset[b'parents'][0], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
353 cset[b'parents'][1], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
354 # Linknode is always itself for changesets. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
355 cset[b'node'], |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
356 # We always send full revisions. So delta base is not set. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
357 nullid, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
358 mdiff.trivialdiffheader(len(data)) + data, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
359 # Flags not yet supported. |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
360 0, |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
361 ) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
362 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
363 added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr), |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
364 addrevisioncb=onchangeset) |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
365 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
366 progress.complete() |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
367 |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
368 return { |
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
369 'added': added, |
39633
ff2de4f2eb3c
exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39631
diff
changeset
|
370 'nodesbyphase': nodesbyphase, |
39635
349482d726ee
exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39633
diff
changeset
|
371 'bookmarks': remotebookmarks, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
372 'manifestnodes': manifestnodes, |
39631
b9e453d683a1
exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39629
diff
changeset
|
373 } |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
374 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
375 def _fetchmanifests(repo, tr, remote, manifestnodes): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
376 rootmanifest = repo.manifestlog.getstorage(b'') |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
377 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
378 # Some manifests can be shared between changesets. Filter out revisions |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
379 # we already know about. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
380 fetchnodes = [] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
381 linkrevs = {} |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
382 seen = set() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
383 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
384 for clrev, node in sorted(manifestnodes.iteritems()): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
385 if node in seen: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
386 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
387 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
388 try: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
389 rootmanifest.rev(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
390 except error.LookupError: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
391 fetchnodes.append(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
392 linkrevs[node] = clrev |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
393 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
394 seen.add(node) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
395 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
396 # TODO handle tree manifests |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
397 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
398 # addgroup() expects 7-tuple describing revisions. This normalizes |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
399 # the wire data to that format. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
400 def iterrevisions(objs, progress): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
401 for manifest in objs: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
402 node = manifest[b'node'] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
403 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
404 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
405 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
406 for field, size in manifest.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
407 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
408 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
409 if b'delta' in extrafields: |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
410 basenode = manifest[b'deltabasenode'] |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
411 delta = extrafields[b'delta'] |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
412 elif b'revision' in extrafields: |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
413 basenode = nullid |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
414 revision = extrafields[b'revision'] |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
415 delta = mdiff.trivialdiffheader(len(revision)) + revision |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
416 else: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
417 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
418 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
419 yield ( |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
420 node, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
421 manifest[b'parents'][0], |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
422 manifest[b'parents'][1], |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
423 # The value passed in is passed to the lookup function passed |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
424 # to addgroup(). We already have a map of manifest node to |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
425 # changelog revision number. So we just pass in the |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
426 # manifest node here and use linkrevs.__getitem__ as the |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
427 # resolution function. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
428 node, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
429 basenode, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
430 delta, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
431 # Flags not yet supported. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
432 0 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
433 ) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
434 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
435 progress.increment() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
436 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
437 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'), |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
438 total=len(fetchnodes)) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
439 |
40173
b797150a1ab9
exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40035
diff
changeset
|
440 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata'] |
b797150a1ab9
exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40035
diff
changeset
|
441 batchsize = commandmeta.get(b'recommendedbatchsize', 10000) |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
442 # TODO make size configurable on client? |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
443 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
444 # We send commands 1 at a time to the remote. This is not the most |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
445 # efficient because we incur a round trip at the end of each batch. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
446 # However, the existing frame-based reactor keeps consuming server |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
447 # data in the background. And this results in response data buffering |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
448 # in memory. This can consume gigabytes of memory. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
449 # TODO send multiple commands in a request once background buffering |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
450 # issues are resolved. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
451 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
452 added = [] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
453 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
454 for i in pycompat.xrange(0, len(fetchnodes), batchsize): |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
455 batch = [node for node in fetchnodes[i:i + batchsize]] |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
456 if not batch: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
457 continue |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
458 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
459 with remote.commandexecutor() as e: |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
460 objs = e.callcommand(b'manifestdata', { |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
461 b'tree': b'', |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
462 b'nodes': batch, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
463 b'fields': {b'parents', b'revision'}, |
39641
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
464 b'haveparents': True, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
465 }).result() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
466 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
467 # Chomp off header object. |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
468 next(objs) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
469 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
470 added.extend(rootmanifest.addgroup( |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
471 iterrevisions(objs, progress), |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
472 linkrevs.__getitem__, |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
473 weakref.proxy(tr))) |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
474 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
475 progress.complete() |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
476 |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
477 return { |
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
478 'added': added, |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
479 'linkrevs': linkrevs, |
39638
d292328e0143
exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39636
diff
changeset
|
480 } |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
481 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
482 def _derivefilesfrommanifests(repo, matcher, manifestnodes): |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
483 """Determine what file nodes are relevant given a set of manifest nodes. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
484 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
485 Returns a dict mapping file paths to dicts of file node to first manifest |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
486 node. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
487 """ |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
488 ml = repo.manifestlog |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
489 fnodes = collections.defaultdict(dict) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
490 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
491 progress = repo.ui.makeprogress( |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
492 _('scanning manifests'), total=len(manifestnodes)) |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
493 |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
494 with progress: |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
495 for manifestnode in manifestnodes: |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
496 m = ml.get(b'', manifestnode) |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
497 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
498 # TODO this will pull in unwanted nodes because it takes the storage |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
499 # delta into consideration. What we really want is something that |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
500 # takes the delta between the manifest's parents. And ideally we |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
501 # would ignore file nodes that are known locally. For now, ignore |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
502 # both these limitations. This will result in incremental fetches |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
503 # requesting data we already have. So this is far from ideal. |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
504 md = m.readfast() |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
505 |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
506 for path, fnode in md.items(): |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
507 if matcher(path): |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
508 fnodes[path].setdefault(fnode, manifestnode) |
40035
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
509 |
7a347d362a45
exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39814
diff
changeset
|
510 progress.increment() |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
511 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
512 return fnodes |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
513 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
514 def _fetchfiles(repo, tr, remote, fnodes, linkrevs): |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
515 """Fetch file data from explicit file revisions.""" |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
516 def iterrevisions(objs, progress): |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
517 for filerevision in objs: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
518 node = filerevision[b'node'] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
519 |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
520 extrafields = {} |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
521 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
522 for field, size in filerevision.get(b'fieldsfollowing', []): |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
523 extrafields[field] = next(objs) |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
524 |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
525 if b'delta' in extrafields: |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
526 basenode = filerevision[b'deltabasenode'] |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
527 delta = extrafields[b'delta'] |
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
528 elif b'revision' in extrafields: |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
529 basenode = nullid |
39814
d059cb669632
wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39641
diff
changeset
|
530 revision = extrafields[b'revision'] |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
531 delta = mdiff.trivialdiffheader(len(revision)) + revision |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
532 else: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
533 continue |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
534 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
535 yield ( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
536 node, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
537 filerevision[b'parents'][0], |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
538 filerevision[b'parents'][1], |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
539 node, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
540 basenode, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
541 delta, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
542 # Flags not yet supported. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
543 0, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
544 ) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
545 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
546 progress.increment() |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
547 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
548 progress = repo.ui.makeprogress( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
549 _('files'), unit=_('chunks'), |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
550 total=sum(len(v) for v in fnodes.itervalues())) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
551 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
552 # TODO make batch size configurable |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
553 batchsize = 10000 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
554 fnodeslist = [x for x in sorted(fnodes.items())] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
555 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
556 for i in pycompat.xrange(0, len(fnodeslist), batchsize): |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
557 batch = [x for x in fnodeslist[i:i + batchsize]] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
558 if not batch: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
559 continue |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
560 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
561 with remote.commandexecutor() as e: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
562 fs = [] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
563 locallinkrevs = {} |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
564 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
565 for path, nodes in batch: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
566 fs.append((path, e.callcommand(b'filedata', { |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
567 b'path': path, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
568 b'nodes': sorted(nodes), |
39641
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
569 b'fields': {b'parents', b'revision'}, |
aa7e312375cf
wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39640
diff
changeset
|
570 b'haveparents': True, |
39640
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
571 }))) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
572 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
573 locallinkrevs[path] = { |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
574 node: linkrevs[manifestnode] |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
575 for node, manifestnode in nodes.iteritems()} |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
576 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
577 for path, f in fs: |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
578 objs = f.result() |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
579 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
580 # Chomp off header objects. |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
581 next(objs) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
582 |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
583 store = repo.file(path) |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
584 store.addgroup( |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
585 iterrevisions(objs, progress), |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
586 locallinkrevs[path].__getitem__, |
039bf1eddc2e
exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39638
diff
changeset
|
587 weakref.proxy(tr)) |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
588 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
589 def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets, |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
590 manlinkrevs, shallow=False): |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
591 """Fetch file data from explicit changeset revisions.""" |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
592 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
593 def iterrevisions(objs, remaining, progress): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
594 while remaining: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
595 filerevision = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
596 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
597 node = filerevision[b'node'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
598 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
599 extrafields = {} |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
600 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
601 for field, size in filerevision.get(b'fieldsfollowing', []): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
602 extrafields[field] = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
603 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
604 if b'delta' in extrafields: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
605 basenode = filerevision[b'deltabasenode'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
606 delta = extrafields[b'delta'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
607 elif b'revision' in extrafields: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
608 basenode = nullid |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
609 revision = extrafields[b'revision'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
610 delta = mdiff.trivialdiffheader(len(revision)) + revision |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
611 else: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
612 continue |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
613 |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
614 if b'linknode' in filerevision: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
615 linknode = filerevision[b'linknode'] |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
616 else: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
617 linknode = node |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
618 |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
619 yield ( |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
620 node, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
621 filerevision[b'parents'][0], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
622 filerevision[b'parents'][1], |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
623 linknode, |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
624 basenode, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
625 delta, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
626 # Flags not yet supported. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
627 0, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
628 ) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
629 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
630 progress.increment() |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
631 remaining -= 1 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
632 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
633 progress = repo.ui.makeprogress( |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
634 _('files'), unit=_('chunks'), |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
635 total=sum(len(v) for v in fnodes.itervalues())) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
636 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
637 commandmeta = remote.apidescriptor[b'commands'][b'filesdata'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
638 batchsize = commandmeta.get(b'recommendedbatchsize', 50000) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
639 |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
640 shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
641 fields = {b'parents', b'revision'} |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
642 clrev = repo.changelog.rev |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
643 |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
644 # There are no guarantees that we'll have ancestor revisions if |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
645 # a) this repo has shallow file storage b) shallow data fetching is enabled. |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
646 # Force remote to not delta against possibly unknown revisions when these |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
647 # conditions hold. |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
648 haveparents = not (shallowfiles or shallow) |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
649 |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
650 # Similarly, we may not have calculated linkrevs for all incoming file |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
651 # revisions. Ask the remote to do work for us in this case. |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
652 if not haveparents: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
653 fields.add(b'linknode') |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
654 |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
655 for i in pycompat.xrange(0, len(csets), batchsize): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
656 batch = [x for x in csets[i:i + batchsize]] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
657 if not batch: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
658 continue |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
659 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
660 with remote.commandexecutor() as e: |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
661 args = { |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
662 b'revisions': [{ |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
663 b'type': b'changesetexplicit', |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
664 b'nodes': batch, |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
665 }], |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
666 b'fields': fields, |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
667 b'haveparents': haveparents, |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
668 } |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
669 |
40327
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
670 if pathfilter: |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
671 args[b'pathfilter'] = pathfilter |
55836a34f41b
exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40179
diff
changeset
|
672 |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
673 objs = e.callcommand(b'filesdata', args).result() |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
674 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
675 # First object is an overall header. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
676 overall = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
677 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
678 # We have overall['totalpaths'] segments. |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
679 for i in pycompat.xrange(overall[b'totalpaths']): |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
680 header = next(objs) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
681 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
682 path = header[b'path'] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
683 store = repo.file(path) |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
684 |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
685 linkrevs = { |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
686 fnode: manlinkrevs[mnode] |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
687 for fnode, mnode in fnodes[path].iteritems()} |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
688 |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
689 def getlinkrev(node): |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
690 if node in linkrevs: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
691 return linkrevs[node] |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
692 else: |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
693 return clrev(node) |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
694 |
40179
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
695 store.addgroup(iterrevisions(objs, header[b'totalitems'], |
b843356d4ae1
exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40176
diff
changeset
|
696 progress), |
40393
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
697 getlinkrev, |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
698 weakref.proxy(tr), |
229d23cdb203
exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40330
diff
changeset
|
699 maybemissingparents=shallow) |