1 # Copyright 2017 Facebook, Inc. |
|
2 # |
|
3 # This software may be used and distributed according to the terms of the |
|
4 # GNU General Public License version 2 or any later version. |
|
5 """ |
|
6 [infinitepushbackup] |
|
7 # Whether to enable automatic backups. If this option is True then a backup |
|
8 # process will be started after every mercurial command that modifies the |
|
9 # repo, for example, commit, amend, histedit, rebase etc. |
|
10 autobackup = False |
|
11 |
|
12 # path to the directory where pushback logs should be stored |
|
13 logdir = path/to/dir |
|
14 |
|
15 # Backup at most maxheadstobackup heads, other heads are ignored. |
|
16 # Negative number means backup everything. |
|
17 maxheadstobackup = -1 |
|
18 |
|
19 # Nodes that should not be backed up. Ancestors of these nodes won't be |
|
20 # backed up either |
|
21 dontbackupnodes = [] |
|
22 |
|
23 # Special option that may be used to trigger re-backuping. For example, |
|
24 # if there was a bug in infinitepush backups, then changing the value of |
|
25 # this option will force all clients to make a "clean" backup |
|
26 backupgeneration = 0 |
|
27 |
|
28 # Hostname value to use. If not specified then socket.gethostname() will |
|
29 # be used |
|
30 hostname = '' |
|
31 |
|
32 # Enable reporting of infinitepush backup status as a summary at the end |
|
33 # of smartlog. |
|
34 enablestatus = False |
|
35 |
|
36 # Whether or not to save information about the latest successful backup. |
|
37 # This information includes the local revision number and unix timestamp |
|
38 # of the last time we successfully made a backup. |
|
39 savelatestbackupinfo = False |
|
40 """ |
|
41 |
|
42 from __future__ import absolute_import |
|
43 |
|
44 import collections |
|
45 import errno |
|
46 import json |
|
47 import os |
|
48 import re |
|
49 import socket |
|
50 import stat |
|
51 import subprocess |
|
52 import time |
|
53 |
|
54 from mercurial.node import ( |
|
55 bin, |
|
56 hex, |
|
57 nullrev, |
|
58 short, |
|
59 ) |
|
60 |
|
61 from mercurial.i18n import _ |
|
62 |
|
63 from mercurial import ( |
|
64 bundle2, |
|
65 changegroup, |
|
66 commands, |
|
67 discovery, |
|
68 dispatch, |
|
69 encoding, |
|
70 error, |
|
71 extensions, |
|
72 hg, |
|
73 localrepo, |
|
74 lock as lockmod, |
|
75 phases, |
|
76 policy, |
|
77 registrar, |
|
78 scmutil, |
|
79 util, |
|
80 ) |
|
81 |
|
82 from . import bundleparts |
|
83 |
|
84 getscratchbookmarkspart = bundleparts.getscratchbookmarkspart |
|
85 getscratchbranchparts = bundleparts.getscratchbranchparts |
|
86 |
|
87 from hgext3rd import shareutil |
|
88 |
|
89 osutil = policy.importmod(r'osutil') |
|
90 |
|
91 cmdtable = {} |
|
92 command = registrar.command(cmdtable) |
|
93 revsetpredicate = registrar.revsetpredicate() |
|
94 templatekeyword = registrar.templatekeyword() |
|
95 |
|
96 backupbookmarktuple = collections.namedtuple('backupbookmarktuple', |
|
97 ['hostname', 'reporoot', 'localbookmark']) |
|
98 |
|
99 class backupstate(object): |
|
100 def __init__(self): |
|
101 self.heads = set() |
|
102 self.localbookmarks = {} |
|
103 |
|
104 def empty(self): |
|
105 return not self.heads and not self.localbookmarks |
|
106 |
|
107 class WrongPermissionsException(Exception): |
|
108 def __init__(self, logdir): |
|
109 self.logdir = logdir |
|
110 |
|
111 restoreoptions = [ |
|
112 ('', 'reporoot', '', 'root of the repo to restore'), |
|
113 ('', 'user', '', 'user who ran the backup'), |
|
114 ('', 'hostname', '', 'hostname of the repo to restore'), |
|
115 ] |
|
116 |
|
117 _backuplockname = 'infinitepushbackup.lock' |
|
118 |
|
119 def extsetup(ui): |
|
120 if ui.configbool('infinitepushbackup', 'autobackup', False): |
|
121 extensions.wrapfunction(dispatch, 'runcommand', |
|
122 _autobackupruncommandwrapper) |
|
123 extensions.wrapfunction(localrepo.localrepository, 'transaction', |
|
124 _transaction) |
|
125 |
|
126 @command('pushbackup', |
|
127 [('', 'background', None, 'run backup in background')]) |
|
128 def backup(ui, repo, dest=None, **opts): |
|
129 """ |
|
130 Pushes commits, bookmarks and heads to infinitepush. |
|
131 New non-extinct commits are saved since the last `hg pushbackup` |
|
132 or since 0 revision if this backup is the first. |
|
133 Local bookmarks are saved remotely as: |
|
134 infinitepush/backups/USERNAME/HOST/REPOROOT/bookmarks/LOCAL_BOOKMARK |
|
135 Local heads are saved remotely as: |
|
136 infinitepush/backups/USERNAME/HOST/REPOROOT/heads/HEAD_HASH |
|
137 """ |
|
138 |
|
139 if opts.get('background'): |
|
140 _dobackgroundbackup(ui, repo, dest) |
|
141 return 0 |
|
142 |
|
143 try: |
|
144 # Wait at most 30 seconds, because that's the average backup time |
|
145 timeout = 30 |
|
146 srcrepo = shareutil.getsrcrepo(repo) |
|
147 with lockmod.lock(srcrepo.vfs, _backuplockname, timeout=timeout): |
|
148 return _dobackup(ui, repo, dest, **opts) |
|
149 except error.LockHeld as e: |
|
150 if e.errno == errno.ETIMEDOUT: |
|
151 ui.warn(_('timeout waiting on backup lock\n')) |
|
152 return 0 |
|
153 else: |
|
154 raise |
|
155 |
|
156 @command('pullbackup', restoreoptions) |
|
157 def restore(ui, repo, dest=None, **opts): |
|
158 """ |
|
159 Pulls commits from infinitepush that were previously saved with |
|
160 `hg pushbackup`. |
|
161 If user has only one backup for the `dest` repo then it will be restored. |
|
162 But user may have backed up many local repos that points to `dest` repo. |
|
163 These local repos may reside on different hosts or in different |
|
164 repo roots. It makes restore ambiguous; `--reporoot` and `--hostname` |
|
165 options are used to disambiguate. |
|
166 """ |
|
167 |
|
168 other = _getremote(repo, ui, dest, **opts) |
|
169 |
|
170 sourcereporoot = opts.get('reporoot') |
|
171 sourcehostname = opts.get('hostname') |
|
172 namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user')) |
|
173 allbackupstates = _downloadbackupstate(ui, other, sourcereporoot, |
|
174 sourcehostname, namingmgr) |
|
175 if len(allbackupstates) == 0: |
|
176 ui.warn(_('no backups found!')) |
|
177 return 1 |
|
178 _checkbackupstates(allbackupstates) |
|
179 |
|
180 __, backupstate = allbackupstates.popitem() |
|
181 pullcmd, pullopts = _getcommandandoptions('^pull') |
|
182 # pull backuped heads and nodes that are pointed by bookmarks |
|
183 pullopts['rev'] = list(backupstate.heads | |
|
184 set(backupstate.localbookmarks.values())) |
|
185 if dest: |
|
186 pullopts['source'] = dest |
|
187 result = pullcmd(ui, repo, **pullopts) |
|
188 |
|
189 with repo.wlock(), repo.lock(), repo.transaction('bookmark') as tr: |
|
190 changes = [] |
|
191 for book, hexnode in backupstate.localbookmarks.iteritems(): |
|
192 if hexnode in repo: |
|
193 changes.append((book, bin(hexnode))) |
|
194 else: |
|
195 ui.warn(_('%s not found, not creating %s bookmark') % |
|
196 (hexnode, book)) |
|
197 repo._bookmarks.applychanges(repo, tr, changes) |
|
198 |
|
199 # manually write local backup state and flag to not autobackup |
|
200 # just after we restored, which would be pointless |
|
201 _writelocalbackupstate(repo.vfs, |
|
202 list(backupstate.heads), |
|
203 backupstate.localbookmarks) |
|
204 repo.ignoreautobackup = True |
|
205 |
|
206 return result |
|
207 |
|
208 @command('getavailablebackups', |
|
209 [('', 'user', '', _('username, defaults to current user')), |
|
210 ('', 'json', None, _('print available backups in json format'))]) |
|
211 def getavailablebackups(ui, repo, dest=None, **opts): |
|
212 other = _getremote(repo, ui, dest, **opts) |
|
213 |
|
214 sourcereporoot = opts.get('reporoot') |
|
215 sourcehostname = opts.get('hostname') |
|
216 |
|
217 namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user')) |
|
218 allbackupstates = _downloadbackupstate(ui, other, sourcereporoot, |
|
219 sourcehostname, namingmgr) |
|
220 |
|
221 if opts.get('json'): |
|
222 jsondict = collections.defaultdict(list) |
|
223 for hostname, reporoot in allbackupstates.keys(): |
|
224 jsondict[hostname].append(reporoot) |
|
225 # make sure the output is sorted. That's not an efficient way to |
|
226 # keep list sorted but we don't have that many backups. |
|
227 jsondict[hostname].sort() |
|
228 ui.write('%s\n' % json.dumps(jsondict)) |
|
229 else: |
|
230 if not allbackupstates: |
|
231 ui.write(_('no backups available for %s\n') % namingmgr.username) |
|
232 |
|
233 ui.write(_('user %s has %d available backups:\n') % |
|
234 (namingmgr.username, len(allbackupstates))) |
|
235 |
|
236 for hostname, reporoot in sorted(allbackupstates.keys()): |
|
237 ui.write(_('%s on %s\n') % (reporoot, hostname)) |
|
238 |
|
239 @command('debugcheckbackup', |
|
240 [('', 'all', None, _('check all backups that user have')), |
|
241 ] + restoreoptions) |
|
242 def checkbackup(ui, repo, dest=None, **opts): |
|
243 """ |
|
244 Checks that all the nodes that backup needs are available in bundlestore |
|
245 This command can check either specific backup (see restoreoptions) or all |
|
246 backups for the user |
|
247 """ |
|
248 |
|
249 sourcereporoot = opts.get('reporoot') |
|
250 sourcehostname = opts.get('hostname') |
|
251 |
|
252 other = _getremote(repo, ui, dest, **opts) |
|
253 namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user')) |
|
254 allbackupstates = _downloadbackupstate(ui, other, sourcereporoot, |
|
255 sourcehostname, namingmgr) |
|
256 if not opts.get('all'): |
|
257 _checkbackupstates(allbackupstates) |
|
258 |
|
259 ret = 0 |
|
260 while allbackupstates: |
|
261 key, bkpstate = allbackupstates.popitem() |
|
262 ui.status(_('checking %s on %s\n') % (key[1], key[0])) |
|
263 if not _dobackupcheck(bkpstate, ui, repo, dest, **opts): |
|
264 ret = 255 |
|
265 return ret |
|
266 |
|
267 @command('debugwaitbackup', [('', 'timeout', '', 'timeout value')]) |
|
268 def waitbackup(ui, repo, timeout): |
|
269 try: |
|
270 if timeout: |
|
271 timeout = int(timeout) |
|
272 else: |
|
273 timeout = -1 |
|
274 except ValueError: |
|
275 raise error.Abort('timeout should be integer') |
|
276 |
|
277 try: |
|
278 repo = shareutil.getsrcrepo(repo) |
|
279 with lockmod.lock(repo.vfs, _backuplockname, timeout=timeout): |
|
280 pass |
|
281 except error.LockHeld as e: |
|
282 if e.errno == errno.ETIMEDOUT: |
|
283 raise error.Abort(_('timeout while waiting for backup')) |
|
284 raise |
|
285 |
|
286 @command('isbackedup', |
|
287 [('r', 'rev', [], _('show the specified revision or revset'), _('REV'))]) |
|
288 def isbackedup(ui, repo, **opts): |
|
289 """checks if commit was backed up to infinitepush |
|
290 |
|
291 If no revision are specified then it checks working copy parent |
|
292 """ |
|
293 |
|
294 revs = opts.get('rev') |
|
295 if not revs: |
|
296 revs = ['.'] |
|
297 bkpstate = _readlocalbackupstate(ui, repo) |
|
298 unfi = repo.unfiltered() |
|
299 backeduprevs = unfi.revs('draft() and ::%ls', bkpstate.heads) |
|
300 for r in scmutil.revrange(unfi, revs): |
|
301 ui.write(_(unfi[r].hex() + ' ')) |
|
302 ui.write(_('backed up' if r in backeduprevs else 'not backed up')) |
|
303 ui.write(_('\n')) |
|
304 |
|
305 @revsetpredicate('backedup') |
|
306 def backedup(repo, subset, x): |
|
307 """Draft changesets that have been backed up by infinitepush""" |
|
308 unfi = repo.unfiltered() |
|
309 bkpstate = _readlocalbackupstate(repo.ui, repo) |
|
310 return subset & unfi.revs('draft() and ::%ls and not hidden()', |
|
311 bkpstate.heads) |
|
312 |
|
313 @revsetpredicate('notbackedup') |
|
314 def notbackedup(repo, subset, x): |
|
315 """Changesets that have not yet been backed up by infinitepush""" |
|
316 bkpstate = _readlocalbackupstate(repo.ui, repo) |
|
317 bkpheads = set(bkpstate.heads) |
|
318 candidates = set(_backupheads(repo.ui, repo)) |
|
319 notbackeduprevs = set() |
|
320 # Find all revisions that are ancestors of the expected backup heads, |
|
321 # stopping when we reach either a public commit or a known backup head. |
|
322 while candidates: |
|
323 candidate = candidates.pop() |
|
324 if candidate not in bkpheads: |
|
325 ctx = repo[candidate] |
|
326 rev = ctx.rev() |
|
327 if rev not in notbackeduprevs and ctx.phase() != phases.public: |
|
328 # This rev may not have been backed up. Record it, and add its |
|
329 # parents as candidates. |
|
330 notbackeduprevs.add(rev) |
|
331 candidates.update([p.hex() for p in ctx.parents()]) |
|
332 if notbackeduprevs: |
|
333 # Some revisions in this set may actually have been backed up by |
|
334 # virtue of being an ancestor of a different backup head, which may |
|
335 # have been hidden since the backup was made. Find these and remove |
|
336 # them from the set. |
|
337 unfi = repo.unfiltered() |
|
338 candidates = bkpheads |
|
339 while candidates: |
|
340 candidate = candidates.pop() |
|
341 if candidate in unfi: |
|
342 ctx = unfi[candidate] |
|
343 if ctx.phase() != phases.public: |
|
344 notbackeduprevs.discard(ctx.rev()) |
|
345 candidates.update([p.hex() for p in ctx.parents()]) |
|
346 return subset & notbackeduprevs |
|
347 |
|
348 @templatekeyword('backingup') |
|
349 def backingup(repo, ctx, **args): |
|
350 """Whether infinitepush is currently backing up commits.""" |
|
351 # If the backup lock exists then a backup should be in progress. |
|
352 srcrepo = shareutil.getsrcrepo(repo) |
|
353 return srcrepo.vfs.lexists(_backuplockname) |
|
354 |
|
355 def smartlogsummary(ui, repo): |
|
356 if not ui.configbool('infinitepushbackup', 'enablestatus'): |
|
357 return |
|
358 |
|
359 # Don't output the summary if a backup is currently in progress. |
|
360 srcrepo = shareutil.getsrcrepo(repo) |
|
361 if srcrepo.vfs.lexists(_backuplockname): |
|
362 return |
|
363 |
|
364 unbackeduprevs = repo.revs('notbackedup()') |
|
365 |
|
366 # Count the number of changesets that haven't been backed up for 10 minutes. |
|
367 # If there is only one, also print out its hash. |
|
368 backuptime = time.time() - 10 * 60 # 10 minutes ago |
|
369 count = 0 |
|
370 singleunbackeduprev = None |
|
371 for rev in unbackeduprevs: |
|
372 if repo[rev].date()[0] <= backuptime: |
|
373 singleunbackeduprev = rev |
|
374 count += 1 |
|
375 if count > 0: |
|
376 if count > 1: |
|
377 ui.warn(_('note: %d changesets are not backed up.\n') % count) |
|
378 else: |
|
379 ui.warn(_('note: changeset %s is not backed up.\n') % |
|
380 short(repo[singleunbackeduprev].node())) |
|
381 ui.warn(_('Run `hg pushbackup` to perform a backup. If this fails,\n' |
|
382 'please report to the Source Control @ FB group.\n')) |
|
383 |
|
384 def _autobackupruncommandwrapper(orig, lui, repo, cmd, fullargs, *args): |
|
385 ''' |
|
386 If this wrapper is enabled then auto backup is started after every command |
|
387 that modifies a repository. |
|
388 Since we don't want to start auto backup after read-only commands, |
|
389 then this wrapper checks if this command opened at least one transaction. |
|
390 If yes then background backup will be started. |
|
391 ''' |
|
392 |
|
393 # For chg, do not wrap the "serve" runcommand call |
|
394 if 'CHGINTERNALMARK' in encoding.environ: |
|
395 return orig(lui, repo, cmd, fullargs, *args) |
|
396 |
|
397 try: |
|
398 return orig(lui, repo, cmd, fullargs, *args) |
|
399 finally: |
|
400 if getattr(repo, 'txnwasopened', False) \ |
|
401 and not getattr(repo, 'ignoreautobackup', False): |
|
402 lui.debug("starting infinitepush autobackup in the background\n") |
|
403 _dobackgroundbackup(lui, repo) |
|
404 |
|
405 def _transaction(orig, self, *args, **kwargs): |
|
406 ''' Wrapper that records if a transaction was opened. |
|
407 |
|
408 If a transaction was opened then we want to start background backup process. |
|
409 This hook records the fact that transaction was opened. |
|
410 ''' |
|
411 self.txnwasopened = True |
|
412 return orig(self, *args, **kwargs) |
|
413 |
|
414 def _backupheads(ui, repo): |
|
415 """Returns the set of heads that should be backed up in this repo.""" |
|
416 maxheadstobackup = ui.configint('infinitepushbackup', |
|
417 'maxheadstobackup', -1) |
|
418 |
|
419 revset = 'heads(draft()) & not obsolete()' |
|
420 |
|
421 backupheads = [ctx.hex() for ctx in repo.set(revset)] |
|
422 if maxheadstobackup > 0: |
|
423 backupheads = backupheads[-maxheadstobackup:] |
|
424 elif maxheadstobackup == 0: |
|
425 backupheads = [] |
|
426 return set(backupheads) |
|
427 |
|
428 def _dobackup(ui, repo, dest, **opts): |
|
429 ui.status(_('starting backup %s\n') % time.strftime('%H:%M:%S %d %b %Y %Z')) |
|
430 start = time.time() |
|
431 # to handle multiple working copies correctly |
|
432 repo = shareutil.getsrcrepo(repo) |
|
433 currentbkpgenerationvalue = _readbackupgenerationfile(repo.vfs) |
|
434 newbkpgenerationvalue = ui.configint('infinitepushbackup', |
|
435 'backupgeneration', 0) |
|
436 if currentbkpgenerationvalue != newbkpgenerationvalue: |
|
437 # Unlinking local backup state will trigger re-backuping |
|
438 _deletebackupstate(repo) |
|
439 _writebackupgenerationfile(repo.vfs, newbkpgenerationvalue) |
|
440 bkpstate = _readlocalbackupstate(ui, repo) |
|
441 |
|
442 # this variable stores the local store info (tip numeric revision and date) |
|
443 # which we use to quickly tell if our backup is stale |
|
444 afterbackupinfo = _getlocalinfo(repo) |
|
445 |
|
446 # This variable will store what heads will be saved in backup state file |
|
447 # if backup finishes successfully |
|
448 afterbackupheads = _backupheads(ui, repo) |
|
449 other = _getremote(repo, ui, dest, **opts) |
|
450 outgoing, badhexnodes = _getrevstobackup(repo, ui, other, |
|
451 afterbackupheads - bkpstate.heads) |
|
452 # If remotefilelog extension is enabled then there can be nodes that we |
|
453 # can't backup. In this case let's remove them from afterbackupheads |
|
454 afterbackupheads.difference_update(badhexnodes) |
|
455 |
|
456 # As afterbackupheads this variable stores what heads will be saved in |
|
457 # backup state file if backup finishes successfully |
|
458 afterbackuplocalbooks = _getlocalbookmarks(repo) |
|
459 afterbackuplocalbooks = _filterbookmarks( |
|
460 afterbackuplocalbooks, repo, afterbackupheads) |
|
461 |
|
462 newheads = afterbackupheads - bkpstate.heads |
|
463 removedheads = bkpstate.heads - afterbackupheads |
|
464 newbookmarks = _dictdiff(afterbackuplocalbooks, bkpstate.localbookmarks) |
|
465 removedbookmarks = _dictdiff(bkpstate.localbookmarks, afterbackuplocalbooks) |
|
466 |
|
467 namingmgr = BackupBookmarkNamingManager(ui, repo) |
|
468 bookmarkstobackup = _getbookmarkstobackup( |
|
469 repo, newbookmarks, removedbookmarks, |
|
470 newheads, removedheads, namingmgr) |
|
471 |
|
472 # Special case if backup state is empty. Clean all backup bookmarks from the |
|
473 # server. |
|
474 if bkpstate.empty(): |
|
475 bookmarkstobackup[namingmgr.getbackupheadprefix()] = '' |
|
476 bookmarkstobackup[namingmgr.getbackupbookmarkprefix()] = '' |
|
477 |
|
478 # Wrap deltaparent function to make sure that bundle takes less space |
|
479 # See _deltaparent comments for details |
|
480 extensions.wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent) |
|
481 try: |
|
482 bundler = _createbundler(ui, repo, other) |
|
483 bundler.addparam("infinitepush", "True") |
|
484 backup = False |
|
485 if outgoing and outgoing.missing: |
|
486 backup = True |
|
487 parts = getscratchbranchparts(repo, other, outgoing, |
|
488 confignonforwardmove=False, |
|
489 ui=ui, bookmark=None, |
|
490 create=False) |
|
491 for part in parts: |
|
492 bundler.addpart(part) |
|
493 |
|
494 if bookmarkstobackup: |
|
495 backup = True |
|
496 bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup)) |
|
497 |
|
498 if backup: |
|
499 _sendbundle(bundler, other) |
|
500 _writelocalbackupstate(repo.vfs, afterbackupheads, |
|
501 afterbackuplocalbooks) |
|
502 if ui.config('infinitepushbackup', 'savelatestbackupinfo'): |
|
503 _writelocalbackupinfo(repo.vfs, **afterbackupinfo) |
|
504 else: |
|
505 ui.status(_('nothing to backup\n')) |
|
506 finally: |
|
507 # cleanup ensures that all pipes are flushed |
|
508 cleanup = getattr(other, '_cleanup', None) or getattr(other, 'cleanup') |
|
509 try: |
|
510 cleanup() |
|
511 except Exception: |
|
512 ui.warn(_('remote connection cleanup failed\n')) |
|
513 ui.status(_('finished in %f seconds\n') % (time.time() - start)) |
|
514 extensions.unwrapfunction(changegroup.cg2packer, 'deltaparent', |
|
515 _deltaparent) |
|
516 return 0 |
|
517 |
|
518 def _dobackgroundbackup(ui, repo, dest=None): |
|
519 background_cmd = ['hg', 'pushbackup'] |
|
520 if dest: |
|
521 background_cmd.append(dest) |
|
522 logfile = None |
|
523 logdir = ui.config('infinitepushbackup', 'logdir') |
|
524 if logdir: |
|
525 # make newly created files and dirs non-writable |
|
526 oldumask = os.umask(0o022) |
|
527 try: |
|
528 try: |
|
529 username = util.shortuser(ui.username()) |
|
530 except Exception: |
|
531 username = 'unknown' |
|
532 |
|
533 if not _checkcommonlogdir(logdir): |
|
534 raise WrongPermissionsException(logdir) |
|
535 |
|
536 userlogdir = os.path.join(logdir, username) |
|
537 util.makedirs(userlogdir) |
|
538 |
|
539 if not _checkuserlogdir(userlogdir): |
|
540 raise WrongPermissionsException(userlogdir) |
|
541 |
|
542 reporoot = repo.origroot |
|
543 reponame = os.path.basename(reporoot) |
|
544 _removeoldlogfiles(userlogdir, reponame) |
|
545 logfile = _getlogfilename(logdir, username, reponame) |
|
546 except (OSError, IOError) as e: |
|
547 ui.debug('infinitepush backup log is disabled: %s\n' % e) |
|
548 except WrongPermissionsException as e: |
|
549 ui.debug(('%s directory has incorrect permission, ' + |
|
550 'infinitepush backup logging will be disabled\n') % |
|
551 e.logdir) |
|
552 finally: |
|
553 os.umask(oldumask) |
|
554 |
|
555 if not logfile: |
|
556 logfile = os.devnull |
|
557 |
|
558 with open(logfile, 'a') as f: |
|
559 subprocess.Popen(background_cmd, shell=False, stdout=f, |
|
560 stderr=subprocess.STDOUT) |
|
561 |
|
562 def _dobackupcheck(bkpstate, ui, repo, dest, **opts): |
|
563 remotehexnodes = sorted( |
|
564 set(bkpstate.heads).union(bkpstate.localbookmarks.values())) |
|
565 if not remotehexnodes: |
|
566 return True |
|
567 other = _getremote(repo, ui, dest, **opts) |
|
568 batch = other.iterbatch() |
|
569 for hexnode in remotehexnodes: |
|
570 batch.lookup(hexnode) |
|
571 batch.submit() |
|
572 lookupresults = batch.results() |
|
573 i = 0 |
|
574 try: |
|
575 for i, r in enumerate(lookupresults): |
|
576 # iterate over results to make it throw if revision |
|
577 # was not found |
|
578 pass |
|
579 return True |
|
580 except error.RepoError: |
|
581 ui.warn(_('unknown revision %r\n') % remotehexnodes[i]) |
|
582 return False |
|
583 |
|
584 _backuplatestinfofile = 'infinitepushlatestbackupinfo' |
|
585 _backupstatefile = 'infinitepushbackupstate' |
|
586 _backupgenerationfile = 'infinitepushbackupgeneration' |
|
587 |
|
588 # Common helper functions |
|
589 def _getlocalinfo(repo): |
|
590 localinfo = {} |
|
591 localinfo['rev'] = repo[repo.changelog.tip()].rev() |
|
592 localinfo['time'] = int(time.time()) |
|
593 return localinfo |
|
594 |
|
595 def _getlocalbookmarks(repo): |
|
596 localbookmarks = {} |
|
597 for bookmark, node in repo._bookmarks.iteritems(): |
|
598 hexnode = hex(node) |
|
599 localbookmarks[bookmark] = hexnode |
|
600 return localbookmarks |
|
601 |
|
602 def _filterbookmarks(localbookmarks, repo, headstobackup): |
|
603 '''Filters out some bookmarks from being backed up |
|
604 |
|
605 Filters out bookmarks that do not point to ancestors of headstobackup or |
|
606 public commits |
|
607 ''' |
|
608 |
|
609 headrevstobackup = [repo[hexhead].rev() for hexhead in headstobackup] |
|
610 ancestors = repo.changelog.ancestors(headrevstobackup, inclusive=True) |
|
611 filteredbooks = {} |
|
612 for bookmark, hexnode in localbookmarks.iteritems(): |
|
613 if (repo[hexnode].rev() in ancestors or |
|
614 repo[hexnode].phase() == phases.public): |
|
615 filteredbooks[bookmark] = hexnode |
|
616 return filteredbooks |
|
617 |
|
618 def _downloadbackupstate(ui, other, sourcereporoot, sourcehostname, namingmgr): |
|
619 pattern = namingmgr.getcommonuserprefix() |
|
620 fetchedbookmarks = other.listkeyspatterns('bookmarks', patterns=[pattern]) |
|
621 allbackupstates = collections.defaultdict(backupstate) |
|
622 for book, hexnode in fetchedbookmarks.iteritems(): |
|
623 parsed = _parsebackupbookmark(book, namingmgr) |
|
624 if parsed: |
|
625 if sourcereporoot and sourcereporoot != parsed.reporoot: |
|
626 continue |
|
627 if sourcehostname and sourcehostname != parsed.hostname: |
|
628 continue |
|
629 key = (parsed.hostname, parsed.reporoot) |
|
630 if parsed.localbookmark: |
|
631 bookname = parsed.localbookmark |
|
632 allbackupstates[key].localbookmarks[bookname] = hexnode |
|
633 else: |
|
634 allbackupstates[key].heads.add(hexnode) |
|
635 else: |
|
636 ui.warn(_('wrong format of backup bookmark: %s') % book) |
|
637 |
|
638 return allbackupstates |
|
639 |
|
640 def _checkbackupstates(allbackupstates): |
|
641 if len(allbackupstates) == 0: |
|
642 raise error.Abort('no backups found!') |
|
643 |
|
644 hostnames = set(key[0] for key in allbackupstates.iterkeys()) |
|
645 reporoots = set(key[1] for key in allbackupstates.iterkeys()) |
|
646 |
|
647 if len(hostnames) > 1: |
|
648 raise error.Abort( |
|
649 _('ambiguous hostname to restore: %s') % sorted(hostnames), |
|
650 hint=_('set --hostname to disambiguate')) |
|
651 |
|
652 if len(reporoots) > 1: |
|
653 raise error.Abort( |
|
654 _('ambiguous repo root to restore: %s') % sorted(reporoots), |
|
655 hint=_('set --reporoot to disambiguate')) |
|
656 |
|
657 class BackupBookmarkNamingManager(object): |
|
658 def __init__(self, ui, repo, username=None): |
|
659 self.ui = ui |
|
660 self.repo = repo |
|
661 if not username: |
|
662 username = util.shortuser(ui.username()) |
|
663 self.username = username |
|
664 |
|
665 self.hostname = self.ui.config('infinitepushbackup', 'hostname') |
|
666 if not self.hostname: |
|
667 self.hostname = socket.gethostname() |
|
668 |
|
669 def getcommonuserprefix(self): |
|
670 return '/'.join((self._getcommonuserprefix(), '*')) |
|
671 |
|
672 def getcommonprefix(self): |
|
673 return '/'.join((self._getcommonprefix(), '*')) |
|
674 |
|
675 def getbackupbookmarkprefix(self): |
|
676 return '/'.join((self._getbackupbookmarkprefix(), '*')) |
|
677 |
|
678 def getbackupbookmarkname(self, bookmark): |
|
679 bookmark = _escapebookmark(bookmark) |
|
680 return '/'.join((self._getbackupbookmarkprefix(), bookmark)) |
|
681 |
|
682 def getbackupheadprefix(self): |
|
683 return '/'.join((self._getbackupheadprefix(), '*')) |
|
684 |
|
685 def getbackupheadname(self, hexhead): |
|
686 return '/'.join((self._getbackupheadprefix(), hexhead)) |
|
687 |
|
688 def _getbackupbookmarkprefix(self): |
|
689 return '/'.join((self._getcommonprefix(), 'bookmarks')) |
|
690 |
|
691 def _getbackupheadprefix(self): |
|
692 return '/'.join((self._getcommonprefix(), 'heads')) |
|
693 |
|
694 def _getcommonuserprefix(self): |
|
695 return '/'.join(('infinitepush', 'backups', self.username)) |
|
696 |
|
697 def _getcommonprefix(self): |
|
698 reporoot = self.repo.origroot |
|
699 |
|
700 result = '/'.join((self._getcommonuserprefix(), self.hostname)) |
|
701 if not reporoot.startswith('/'): |
|
702 result += '/' |
|
703 result += reporoot |
|
704 if result.endswith('/'): |
|
705 result = result[:-1] |
|
706 return result |
|
707 |
|
708 def _escapebookmark(bookmark): |
|
709 ''' |
|
710 If `bookmark` contains "bookmarks" as a substring then replace it with |
|
711 "bookmarksbookmarks". This will make parsing remote bookmark name |
|
712 unambigious. |
|
713 ''' |
|
714 |
|
715 bookmark = encoding.fromlocal(bookmark) |
|
716 return bookmark.replace('bookmarks', 'bookmarksbookmarks') |
|
717 |
|
718 def _unescapebookmark(bookmark): |
|
719 bookmark = encoding.tolocal(bookmark) |
|
720 return bookmark.replace('bookmarksbookmarks', 'bookmarks') |
|
721 |
|
722 def _getremote(repo, ui, dest, **opts): |
|
723 path = ui.paths.getpath(dest, default=('infinitepush', 'default')) |
|
724 if not path: |
|
725 raise error.Abort(_('default repository not configured!'), |
|
726 hint=_("see 'hg help config.paths'")) |
|
727 dest = path.pushloc or path.loc |
|
728 return hg.peer(repo, opts, dest) |
|
729 |
|
730 def _getcommandandoptions(command): |
|
731 cmd = commands.table[command][0] |
|
732 opts = dict(opt[1:3] for opt in commands.table[command][1]) |
|
733 return cmd, opts |
|
734 |
|
735 # Backup helper functions |
|
736 |
|
737 def _deltaparent(orig, self, revlog, rev, p1, p2, prev): |
|
738 # This version of deltaparent prefers p1 over prev to use less space |
|
739 dp = revlog.deltaparent(rev) |
|
740 if dp == nullrev and not revlog.storedeltachains: |
|
741 # send full snapshot only if revlog configured to do so |
|
742 return nullrev |
|
743 return p1 |
|
744 |
|
745 def _getbookmarkstobackup(repo, newbookmarks, removedbookmarks, |
|
746 newheads, removedheads, namingmgr): |
|
747 bookmarkstobackup = {} |
|
748 |
|
749 for bookmark, hexnode in removedbookmarks.items(): |
|
750 backupbookmark = namingmgr.getbackupbookmarkname(bookmark) |
|
751 bookmarkstobackup[backupbookmark] = '' |
|
752 |
|
753 for bookmark, hexnode in newbookmarks.items(): |
|
754 backupbookmark = namingmgr.getbackupbookmarkname(bookmark) |
|
755 bookmarkstobackup[backupbookmark] = hexnode |
|
756 |
|
757 for hexhead in removedheads: |
|
758 headbookmarksname = namingmgr.getbackupheadname(hexhead) |
|
759 bookmarkstobackup[headbookmarksname] = '' |
|
760 |
|
761 for hexhead in newheads: |
|
762 headbookmarksname = namingmgr.getbackupheadname(hexhead) |
|
763 bookmarkstobackup[headbookmarksname] = hexhead |
|
764 |
|
765 return bookmarkstobackup |
|
766 |
|
767 def _createbundler(ui, repo, other): |
|
768 bundler = bundle2.bundle20(ui, bundle2.bundle2caps(other)) |
|
769 # Disallow pushback because we want to avoid taking repo locks. |
|
770 # And we don't need pushback anyway |
|
771 capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, |
|
772 allowpushback=False)) |
|
773 bundler.newpart('replycaps', data=capsblob) |
|
774 return bundler |
|
775 |
|
776 def _sendbundle(bundler, other): |
|
777 stream = util.chunkbuffer(bundler.getchunks()) |
|
778 try: |
|
779 other.unbundle(stream, ['force'], other.url()) |
|
780 except error.BundleValueError as exc: |
|
781 raise error.Abort(_('missing support for %s') % exc) |
|
782 |
|
783 def findcommonoutgoing(repo, ui, other, heads): |
|
784 if heads: |
|
785 # Avoid using remotenames fastheaddiscovery heuristic. It uses |
|
786 # remotenames file to quickly find commonoutgoing set, but it can |
|
787 # result in sending public commits to infinitepush servers. |
|
788 # For example: |
|
789 # |
|
790 # o draft |
|
791 # / |
|
792 # o C1 |
|
793 # | |
|
794 # ... |
|
795 # | |
|
796 # o remote/master |
|
797 # |
|
798 # pushbackup in that case results in sending to the infinitepush server |
|
799 # all public commits from 'remote/master' to C1. It increases size of |
|
800 # the bundle + it may result in storing data about public commits |
|
801 # in infinitepush table. |
|
802 |
|
803 with ui.configoverride({("remotenames", "fastheaddiscovery"): False}): |
|
804 nodes = map(repo.changelog.node, heads) |
|
805 return discovery.findcommonoutgoing(repo, other, onlyheads=nodes) |
|
806 else: |
|
807 return None |
|
808 |
|
809 def _getrevstobackup(repo, ui, other, headstobackup): |
|
810 # In rare cases it's possible to have a local node without filelogs. |
|
811 # This is possible if remotefilelog is enabled and if the node was |
|
812 # stripped server-side. We want to filter out these bad nodes and all |
|
813 # of their descendants. |
|
814 badnodes = ui.configlist('infinitepushbackup', 'dontbackupnodes', []) |
|
815 badnodes = [node for node in badnodes if node in repo] |
|
816 badrevs = [repo[node].rev() for node in badnodes] |
|
817 badnodesdescendants = repo.set('%ld::', badrevs) if badrevs else set() |
|
818 badnodesdescendants = set(ctx.hex() for ctx in badnodesdescendants) |
|
819 filteredheads = filter(lambda head: head in badnodesdescendants, |
|
820 headstobackup) |
|
821 |
|
822 if filteredheads: |
|
823 ui.warn(_('filtering nodes: %s\n') % filteredheads) |
|
824 ui.log('infinitepushbackup', 'corrupted nodes found', |
|
825 infinitepushbackupcorruptednodes='failure') |
|
826 headstobackup = filter(lambda head: head not in badnodesdescendants, |
|
827 headstobackup) |
|
828 |
|
829 revs = list(repo[hexnode].rev() for hexnode in headstobackup) |
|
830 outgoing = findcommonoutgoing(repo, ui, other, revs) |
|
831 nodeslimit = 1000 |
|
832 if outgoing and len(outgoing.missing) > nodeslimit: |
|
833 # trying to push too many nodes usually means that there is a bug |
|
834 # somewhere. Let's be safe and avoid pushing too many nodes at once |
|
835 raise error.Abort('trying to back up too many nodes: %d' % |
|
836 (len(outgoing.missing),)) |
|
837 return outgoing, set(filteredheads) |
|
838 |
|
839 def _localbackupstateexists(repo): |
|
840 return repo.vfs.exists(_backupstatefile) |
|
841 |
|
842 def _deletebackupstate(repo): |
|
843 return repo.vfs.tryunlink(_backupstatefile) |
|
844 |
|
845 def _readlocalbackupstate(ui, repo): |
|
846 repo = shareutil.getsrcrepo(repo) |
|
847 if not _localbackupstateexists(repo): |
|
848 return backupstate() |
|
849 |
|
850 with repo.vfs(_backupstatefile) as f: |
|
851 try: |
|
852 state = json.loads(f.read()) |
|
853 if (not isinstance(state['bookmarks'], dict) or |
|
854 not isinstance(state['heads'], list)): |
|
855 raise ValueError('bad types of bookmarks or heads') |
|
856 |
|
857 result = backupstate() |
|
858 result.heads = set(map(str, state['heads'])) |
|
859 result.localbookmarks = state['bookmarks'] |
|
860 return result |
|
861 except (ValueError, KeyError, TypeError) as e: |
|
862 ui.warn(_('corrupt file: %s (%s)\n') % (_backupstatefile, e)) |
|
863 return backupstate() |
|
864 return backupstate() |
|
865 |
|
866 def _writelocalbackupstate(vfs, heads, bookmarks): |
|
867 with vfs(_backupstatefile, 'w') as f: |
|
868 f.write(json.dumps({'heads': list(heads), 'bookmarks': bookmarks})) |
|
869 |
|
870 def _readbackupgenerationfile(vfs): |
|
871 try: |
|
872 with vfs(_backupgenerationfile) as f: |
|
873 return int(f.read()) |
|
874 except (IOError, OSError, ValueError): |
|
875 return 0 |
|
876 |
|
877 def _writebackupgenerationfile(vfs, backupgenerationvalue): |
|
878 with vfs(_backupgenerationfile, 'w', atomictemp=True) as f: |
|
879 f.write(str(backupgenerationvalue)) |
|
880 |
|
881 def _writelocalbackupinfo(vfs, rev, time): |
|
882 with vfs(_backuplatestinfofile, 'w', atomictemp=True) as f: |
|
883 f.write(('backuprevision=%d\nbackuptime=%d\n') % (rev, time)) |
|
884 |
|
885 # Restore helper functions |
|
886 def _parsebackupbookmark(backupbookmark, namingmgr): |
|
887 '''Parses backup bookmark and returns info about it |
|
888 |
|
889 Backup bookmark may represent either a local bookmark or a head. |
|
890 Returns None if backup bookmark has wrong format or tuple. |
|
891 First entry is a hostname where this bookmark came from. |
|
892 Second entry is a root of the repo where this bookmark came from. |
|
893 Third entry in a tuple is local bookmark if backup bookmark |
|
894 represents a local bookmark and None otherwise. |
|
895 ''' |
|
896 |
|
897 backupbookmarkprefix = namingmgr._getcommonuserprefix() |
|
898 commonre = '^{0}/([-\w.]+)(/.*)'.format(re.escape(backupbookmarkprefix)) |
|
899 bookmarkre = commonre + '/bookmarks/(.*)$' |
|
900 headsre = commonre + '/heads/[a-f0-9]{40}$' |
|
901 |
|
902 match = re.search(bookmarkre, backupbookmark) |
|
903 if not match: |
|
904 match = re.search(headsre, backupbookmark) |
|
905 if not match: |
|
906 return None |
|
907 # It's a local head not a local bookmark. |
|
908 # That's why localbookmark is None |
|
909 return backupbookmarktuple(hostname=match.group(1), |
|
910 reporoot=match.group(2), |
|
911 localbookmark=None) |
|
912 |
|
913 return backupbookmarktuple(hostname=match.group(1), |
|
914 reporoot=match.group(2), |
|
915 localbookmark=_unescapebookmark(match.group(3))) |
|
916 |
|
917 _timeformat = '%Y%m%d' |
|
918 |
|
919 def _getlogfilename(logdir, username, reponame): |
|
920 '''Returns name of the log file for particular user and repo |
|
921 |
|
922 Different users have different directories inside logdir. Log filename |
|
923 consists of reponame (basename of repo path) and current day |
|
924 (see _timeformat). That means that two different repos with the same name |
|
925 can share the same log file. This is not a big problem so we ignore it. |
|
926 ''' |
|
927 |
|
928 currentday = time.strftime(_timeformat) |
|
929 return os.path.join(logdir, username, reponame + currentday) |
|
930 |
|
931 def _removeoldlogfiles(userlogdir, reponame): |
|
932 existinglogfiles = [] |
|
933 for entry in osutil.listdir(userlogdir): |
|
934 filename = entry[0] |
|
935 fullpath = os.path.join(userlogdir, filename) |
|
936 if filename.startswith(reponame) and os.path.isfile(fullpath): |
|
937 try: |
|
938 time.strptime(filename[len(reponame):], _timeformat) |
|
939 except ValueError: |
|
940 continue |
|
941 existinglogfiles.append(filename) |
|
942 |
|
943 # _timeformat gives us a property that if we sort log file names in |
|
944 # descending order then newer files are going to be in the beginning |
|
945 existinglogfiles = sorted(existinglogfiles, reverse=True) |
|
946 # Delete logs that are older than 5 days |
|
947 maxlogfilenumber = 5 |
|
948 if len(existinglogfiles) > maxlogfilenumber: |
|
949 for filename in existinglogfiles[maxlogfilenumber:]: |
|
950 os.unlink(os.path.join(userlogdir, filename)) |
|
951 |
|
952 def _checkcommonlogdir(logdir): |
|
953 '''Checks permissions of the log directory |
|
954 |
|
955 We want log directory to actually be a directory, have restricting |
|
956 deletion flag set (sticky bit) |
|
957 ''' |
|
958 |
|
959 try: |
|
960 st = os.stat(logdir) |
|
961 return stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX |
|
962 except OSError: |
|
963 # is raised by os.stat() |
|
964 return False |
|
965 |
|
966 def _checkuserlogdir(userlogdir): |
|
967 '''Checks permissions of the user log directory |
|
968 |
|
969 We want user log directory to be writable only by the user who created it |
|
970 and be owned by `username` |
|
971 ''' |
|
972 |
|
973 try: |
|
974 st = os.stat(userlogdir) |
|
975 # Check that `userlogdir` is owned by `username` |
|
976 if os.getuid() != st.st_uid: |
|
977 return False |
|
978 return ((st.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)) == |
|
979 stat.S_IWUSR) |
|
980 except OSError: |
|
981 # is raised by os.stat() |
|
982 return False |
|
983 |
|
984 def _dictdiff(first, second): |
|
985 '''Returns new dict that contains items from the first dict that are missing |
|
986 from the second dict. |
|
987 ''' |
|
988 result = {} |
|
989 for book, hexnode in first.items(): |
|
990 if second.get(book) != hexnode: |
|
991 result[book] = hexnode |
|
992 return result |
|