--- a/hgext/infinitepush/backupcommands.py Fri Feb 09 13:39:15 2018 +0530
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,992 +0,0 @@
-# Copyright 2017 Facebook, Inc.
-#
-# This software may be used and distributed according to the terms of the
-# GNU General Public License version 2 or any later version.
-"""
- [infinitepushbackup]
- # Whether to enable automatic backups. If this option is True then a backup
- # process will be started after every mercurial command that modifies the
- # repo, for example, commit, amend, histedit, rebase etc.
- autobackup = False
-
- # path to the directory where pushback logs should be stored
- logdir = path/to/dir
-
- # Backup at most maxheadstobackup heads, other heads are ignored.
- # Negative number means backup everything.
- maxheadstobackup = -1
-
- # Nodes that should not be backed up. Ancestors of these nodes won't be
- # backed up either
- dontbackupnodes = []
-
- # Special option that may be used to trigger re-backuping. For example,
- # if there was a bug in infinitepush backups, then changing the value of
- # this option will force all clients to make a "clean" backup
- backupgeneration = 0
-
- # Hostname value to use. If not specified then socket.gethostname() will
- # be used
- hostname = ''
-
- # Enable reporting of infinitepush backup status as a summary at the end
- # of smartlog.
- enablestatus = False
-
- # Whether or not to save information about the latest successful backup.
- # This information includes the local revision number and unix timestamp
- # of the last time we successfully made a backup.
- savelatestbackupinfo = False
-"""
-
-from __future__ import absolute_import
-
-import collections
-import errno
-import json
-import os
-import re
-import socket
-import stat
-import subprocess
-import time
-
-from mercurial.node import (
- bin,
- hex,
- nullrev,
- short,
-)
-
-from mercurial.i18n import _
-
-from mercurial import (
- bundle2,
- changegroup,
- commands,
- discovery,
- dispatch,
- encoding,
- error,
- extensions,
- hg,
- localrepo,
- lock as lockmod,
- phases,
- policy,
- registrar,
- scmutil,
- util,
-)
-
-from . import bundleparts
-
-getscratchbookmarkspart = bundleparts.getscratchbookmarkspart
-getscratchbranchparts = bundleparts.getscratchbranchparts
-
-from hgext3rd import shareutil
-
-osutil = policy.importmod(r'osutil')
-
-cmdtable = {}
-command = registrar.command(cmdtable)
-revsetpredicate = registrar.revsetpredicate()
-templatekeyword = registrar.templatekeyword()
-
-backupbookmarktuple = collections.namedtuple('backupbookmarktuple',
- ['hostname', 'reporoot', 'localbookmark'])
-
-class backupstate(object):
- def __init__(self):
- self.heads = set()
- self.localbookmarks = {}
-
- def empty(self):
- return not self.heads and not self.localbookmarks
-
-class WrongPermissionsException(Exception):
- def __init__(self, logdir):
- self.logdir = logdir
-
-restoreoptions = [
- ('', 'reporoot', '', 'root of the repo to restore'),
- ('', 'user', '', 'user who ran the backup'),
- ('', 'hostname', '', 'hostname of the repo to restore'),
-]
-
-_backuplockname = 'infinitepushbackup.lock'
-
-def extsetup(ui):
- if ui.configbool('infinitepushbackup', 'autobackup', False):
- extensions.wrapfunction(dispatch, 'runcommand',
- _autobackupruncommandwrapper)
- extensions.wrapfunction(localrepo.localrepository, 'transaction',
- _transaction)
-
-@command('pushbackup',
- [('', 'background', None, 'run backup in background')])
-def backup(ui, repo, dest=None, **opts):
- """
- Pushes commits, bookmarks and heads to infinitepush.
- New non-extinct commits are saved since the last `hg pushbackup`
- or since 0 revision if this backup is the first.
- Local bookmarks are saved remotely as:
- infinitepush/backups/USERNAME/HOST/REPOROOT/bookmarks/LOCAL_BOOKMARK
- Local heads are saved remotely as:
- infinitepush/backups/USERNAME/HOST/REPOROOT/heads/HEAD_HASH
- """
-
- if opts.get('background'):
- _dobackgroundbackup(ui, repo, dest)
- return 0
-
- try:
- # Wait at most 30 seconds, because that's the average backup time
- timeout = 30
- srcrepo = shareutil.getsrcrepo(repo)
- with lockmod.lock(srcrepo.vfs, _backuplockname, timeout=timeout):
- return _dobackup(ui, repo, dest, **opts)
- except error.LockHeld as e:
- if e.errno == errno.ETIMEDOUT:
- ui.warn(_('timeout waiting on backup lock\n'))
- return 0
- else:
- raise
-
-@command('pullbackup', restoreoptions)
-def restore(ui, repo, dest=None, **opts):
- """
- Pulls commits from infinitepush that were previously saved with
- `hg pushbackup`.
- If user has only one backup for the `dest` repo then it will be restored.
- But user may have backed up many local repos that points to `dest` repo.
- These local repos may reside on different hosts or in different
- repo roots. It makes restore ambiguous; `--reporoot` and `--hostname`
- options are used to disambiguate.
- """
-
- other = _getremote(repo, ui, dest, **opts)
-
- sourcereporoot = opts.get('reporoot')
- sourcehostname = opts.get('hostname')
- namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
- allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
- sourcehostname, namingmgr)
- if len(allbackupstates) == 0:
- ui.warn(_('no backups found!'))
- return 1
- _checkbackupstates(allbackupstates)
-
- __, backupstate = allbackupstates.popitem()
- pullcmd, pullopts = _getcommandandoptions('^pull')
- # pull backuped heads and nodes that are pointed by bookmarks
- pullopts['rev'] = list(backupstate.heads |
- set(backupstate.localbookmarks.values()))
- if dest:
- pullopts['source'] = dest
- result = pullcmd(ui, repo, **pullopts)
-
- with repo.wlock(), repo.lock(), repo.transaction('bookmark') as tr:
- changes = []
- for book, hexnode in backupstate.localbookmarks.iteritems():
- if hexnode in repo:
- changes.append((book, bin(hexnode)))
- else:
- ui.warn(_('%s not found, not creating %s bookmark') %
- (hexnode, book))
- repo._bookmarks.applychanges(repo, tr, changes)
-
- # manually write local backup state and flag to not autobackup
- # just after we restored, which would be pointless
- _writelocalbackupstate(repo.vfs,
- list(backupstate.heads),
- backupstate.localbookmarks)
- repo.ignoreautobackup = True
-
- return result
-
-@command('getavailablebackups',
- [('', 'user', '', _('username, defaults to current user')),
- ('', 'json', None, _('print available backups in json format'))])
-def getavailablebackups(ui, repo, dest=None, **opts):
- other = _getremote(repo, ui, dest, **opts)
-
- sourcereporoot = opts.get('reporoot')
- sourcehostname = opts.get('hostname')
-
- namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
- allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
- sourcehostname, namingmgr)
-
- if opts.get('json'):
- jsondict = collections.defaultdict(list)
- for hostname, reporoot in allbackupstates.keys():
- jsondict[hostname].append(reporoot)
- # make sure the output is sorted. That's not an efficient way to
- # keep list sorted but we don't have that many backups.
- jsondict[hostname].sort()
- ui.write('%s\n' % json.dumps(jsondict))
- else:
- if not allbackupstates:
- ui.write(_('no backups available for %s\n') % namingmgr.username)
-
- ui.write(_('user %s has %d available backups:\n') %
- (namingmgr.username, len(allbackupstates)))
-
- for hostname, reporoot in sorted(allbackupstates.keys()):
- ui.write(_('%s on %s\n') % (reporoot, hostname))
-
-@command('debugcheckbackup',
- [('', 'all', None, _('check all backups that user have')),
- ] + restoreoptions)
-def checkbackup(ui, repo, dest=None, **opts):
- """
- Checks that all the nodes that backup needs are available in bundlestore
- This command can check either specific backup (see restoreoptions) or all
- backups for the user
- """
-
- sourcereporoot = opts.get('reporoot')
- sourcehostname = opts.get('hostname')
-
- other = _getremote(repo, ui, dest, **opts)
- namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
- allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
- sourcehostname, namingmgr)
- if not opts.get('all'):
- _checkbackupstates(allbackupstates)
-
- ret = 0
- while allbackupstates:
- key, bkpstate = allbackupstates.popitem()
- ui.status(_('checking %s on %s\n') % (key[1], key[0]))
- if not _dobackupcheck(bkpstate, ui, repo, dest, **opts):
- ret = 255
- return ret
-
-@command('debugwaitbackup', [('', 'timeout', '', 'timeout value')])
-def waitbackup(ui, repo, timeout):
- try:
- if timeout:
- timeout = int(timeout)
- else:
- timeout = -1
- except ValueError:
- raise error.Abort('timeout should be integer')
-
- try:
- repo = shareutil.getsrcrepo(repo)
- with lockmod.lock(repo.vfs, _backuplockname, timeout=timeout):
- pass
- except error.LockHeld as e:
- if e.errno == errno.ETIMEDOUT:
- raise error.Abort(_('timeout while waiting for backup'))
- raise
-
-@command('isbackedup',
- [('r', 'rev', [], _('show the specified revision or revset'), _('REV'))])
-def isbackedup(ui, repo, **opts):
- """checks if commit was backed up to infinitepush
-
- If no revision are specified then it checks working copy parent
- """
-
- revs = opts.get('rev')
- if not revs:
- revs = ['.']
- bkpstate = _readlocalbackupstate(ui, repo)
- unfi = repo.unfiltered()
- backeduprevs = unfi.revs('draft() and ::%ls', bkpstate.heads)
- for r in scmutil.revrange(unfi, revs):
- ui.write(_(unfi[r].hex() + ' '))
- ui.write(_('backed up' if r in backeduprevs else 'not backed up'))
- ui.write(_('\n'))
-
-@revsetpredicate('backedup')
-def backedup(repo, subset, x):
- """Draft changesets that have been backed up by infinitepush"""
- unfi = repo.unfiltered()
- bkpstate = _readlocalbackupstate(repo.ui, repo)
- return subset & unfi.revs('draft() and ::%ls and not hidden()',
- bkpstate.heads)
-
-@revsetpredicate('notbackedup')
-def notbackedup(repo, subset, x):
- """Changesets that have not yet been backed up by infinitepush"""
- bkpstate = _readlocalbackupstate(repo.ui, repo)
- bkpheads = set(bkpstate.heads)
- candidates = set(_backupheads(repo.ui, repo))
- notbackeduprevs = set()
- # Find all revisions that are ancestors of the expected backup heads,
- # stopping when we reach either a public commit or a known backup head.
- while candidates:
- candidate = candidates.pop()
- if candidate not in bkpheads:
- ctx = repo[candidate]
- rev = ctx.rev()
- if rev not in notbackeduprevs and ctx.phase() != phases.public:
- # This rev may not have been backed up. Record it, and add its
- # parents as candidates.
- notbackeduprevs.add(rev)
- candidates.update([p.hex() for p in ctx.parents()])
- if notbackeduprevs:
- # Some revisions in this set may actually have been backed up by
- # virtue of being an ancestor of a different backup head, which may
- # have been hidden since the backup was made. Find these and remove
- # them from the set.
- unfi = repo.unfiltered()
- candidates = bkpheads
- while candidates:
- candidate = candidates.pop()
- if candidate in unfi:
- ctx = unfi[candidate]
- if ctx.phase() != phases.public:
- notbackeduprevs.discard(ctx.rev())
- candidates.update([p.hex() for p in ctx.parents()])
- return subset & notbackeduprevs
-
-@templatekeyword('backingup')
-def backingup(repo, ctx, **args):
- """Whether infinitepush is currently backing up commits."""
- # If the backup lock exists then a backup should be in progress.
- srcrepo = shareutil.getsrcrepo(repo)
- return srcrepo.vfs.lexists(_backuplockname)
-
-def smartlogsummary(ui, repo):
- if not ui.configbool('infinitepushbackup', 'enablestatus'):
- return
-
- # Don't output the summary if a backup is currently in progress.
- srcrepo = shareutil.getsrcrepo(repo)
- if srcrepo.vfs.lexists(_backuplockname):
- return
-
- unbackeduprevs = repo.revs('notbackedup()')
-
- # Count the number of changesets that haven't been backed up for 10 minutes.
- # If there is only one, also print out its hash.
- backuptime = time.time() - 10 * 60 # 10 minutes ago
- count = 0
- singleunbackeduprev = None
- for rev in unbackeduprevs:
- if repo[rev].date()[0] <= backuptime:
- singleunbackeduprev = rev
- count += 1
- if count > 0:
- if count > 1:
- ui.warn(_('note: %d changesets are not backed up.\n') % count)
- else:
- ui.warn(_('note: changeset %s is not backed up.\n') %
- short(repo[singleunbackeduprev].node()))
- ui.warn(_('Run `hg pushbackup` to perform a backup. If this fails,\n'
- 'please report to the Source Control @ FB group.\n'))
-
-def _autobackupruncommandwrapper(orig, lui, repo, cmd, fullargs, *args):
- '''
- If this wrapper is enabled then auto backup is started after every command
- that modifies a repository.
- Since we don't want to start auto backup after read-only commands,
- then this wrapper checks if this command opened at least one transaction.
- If yes then background backup will be started.
- '''
-
- # For chg, do not wrap the "serve" runcommand call
- if 'CHGINTERNALMARK' in encoding.environ:
- return orig(lui, repo, cmd, fullargs, *args)
-
- try:
- return orig(lui, repo, cmd, fullargs, *args)
- finally:
- if getattr(repo, 'txnwasopened', False) \
- and not getattr(repo, 'ignoreautobackup', False):
- lui.debug("starting infinitepush autobackup in the background\n")
- _dobackgroundbackup(lui, repo)
-
-def _transaction(orig, self, *args, **kwargs):
- ''' Wrapper that records if a transaction was opened.
-
- If a transaction was opened then we want to start background backup process.
- This hook records the fact that transaction was opened.
- '''
- self.txnwasopened = True
- return orig(self, *args, **kwargs)
-
-def _backupheads(ui, repo):
- """Returns the set of heads that should be backed up in this repo."""
- maxheadstobackup = ui.configint('infinitepushbackup',
- 'maxheadstobackup', -1)
-
- revset = 'heads(draft()) & not obsolete()'
-
- backupheads = [ctx.hex() for ctx in repo.set(revset)]
- if maxheadstobackup > 0:
- backupheads = backupheads[-maxheadstobackup:]
- elif maxheadstobackup == 0:
- backupheads = []
- return set(backupheads)
-
-def _dobackup(ui, repo, dest, **opts):
- ui.status(_('starting backup %s\n') % time.strftime('%H:%M:%S %d %b %Y %Z'))
- start = time.time()
- # to handle multiple working copies correctly
- repo = shareutil.getsrcrepo(repo)
- currentbkpgenerationvalue = _readbackupgenerationfile(repo.vfs)
- newbkpgenerationvalue = ui.configint('infinitepushbackup',
- 'backupgeneration', 0)
- if currentbkpgenerationvalue != newbkpgenerationvalue:
- # Unlinking local backup state will trigger re-backuping
- _deletebackupstate(repo)
- _writebackupgenerationfile(repo.vfs, newbkpgenerationvalue)
- bkpstate = _readlocalbackupstate(ui, repo)
-
- # this variable stores the local store info (tip numeric revision and date)
- # which we use to quickly tell if our backup is stale
- afterbackupinfo = _getlocalinfo(repo)
-
- # This variable will store what heads will be saved in backup state file
- # if backup finishes successfully
- afterbackupheads = _backupheads(ui, repo)
- other = _getremote(repo, ui, dest, **opts)
- outgoing, badhexnodes = _getrevstobackup(repo, ui, other,
- afterbackupheads - bkpstate.heads)
- # If remotefilelog extension is enabled then there can be nodes that we
- # can't backup. In this case let's remove them from afterbackupheads
- afterbackupheads.difference_update(badhexnodes)
-
- # As afterbackupheads this variable stores what heads will be saved in
- # backup state file if backup finishes successfully
- afterbackuplocalbooks = _getlocalbookmarks(repo)
- afterbackuplocalbooks = _filterbookmarks(
- afterbackuplocalbooks, repo, afterbackupheads)
-
- newheads = afterbackupheads - bkpstate.heads
- removedheads = bkpstate.heads - afterbackupheads
- newbookmarks = _dictdiff(afterbackuplocalbooks, bkpstate.localbookmarks)
- removedbookmarks = _dictdiff(bkpstate.localbookmarks, afterbackuplocalbooks)
-
- namingmgr = BackupBookmarkNamingManager(ui, repo)
- bookmarkstobackup = _getbookmarkstobackup(
- repo, newbookmarks, removedbookmarks,
- newheads, removedheads, namingmgr)
-
- # Special case if backup state is empty. Clean all backup bookmarks from the
- # server.
- if bkpstate.empty():
- bookmarkstobackup[namingmgr.getbackupheadprefix()] = ''
- bookmarkstobackup[namingmgr.getbackupbookmarkprefix()] = ''
-
- # Wrap deltaparent function to make sure that bundle takes less space
- # See _deltaparent comments for details
- extensions.wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent)
- try:
- bundler = _createbundler(ui, repo, other)
- bundler.addparam("infinitepush", "True")
- backup = False
- if outgoing and outgoing.missing:
- backup = True
- parts = getscratchbranchparts(repo, other, outgoing,
- confignonforwardmove=False,
- ui=ui, bookmark=None,
- create=False)
- for part in parts:
- bundler.addpart(part)
-
- if bookmarkstobackup:
- backup = True
- bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup))
-
- if backup:
- _sendbundle(bundler, other)
- _writelocalbackupstate(repo.vfs, afterbackupheads,
- afterbackuplocalbooks)
- if ui.config('infinitepushbackup', 'savelatestbackupinfo'):
- _writelocalbackupinfo(repo.vfs, **afterbackupinfo)
- else:
- ui.status(_('nothing to backup\n'))
- finally:
- # cleanup ensures that all pipes are flushed
- cleanup = getattr(other, '_cleanup', None) or getattr(other, 'cleanup')
- try:
- cleanup()
- except Exception:
- ui.warn(_('remote connection cleanup failed\n'))
- ui.status(_('finished in %f seconds\n') % (time.time() - start))
- extensions.unwrapfunction(changegroup.cg2packer, 'deltaparent',
- _deltaparent)
- return 0
-
-def _dobackgroundbackup(ui, repo, dest=None):
- background_cmd = ['hg', 'pushbackup']
- if dest:
- background_cmd.append(dest)
- logfile = None
- logdir = ui.config('infinitepushbackup', 'logdir')
- if logdir:
- # make newly created files and dirs non-writable
- oldumask = os.umask(0o022)
- try:
- try:
- username = util.shortuser(ui.username())
- except Exception:
- username = 'unknown'
-
- if not _checkcommonlogdir(logdir):
- raise WrongPermissionsException(logdir)
-
- userlogdir = os.path.join(logdir, username)
- util.makedirs(userlogdir)
-
- if not _checkuserlogdir(userlogdir):
- raise WrongPermissionsException(userlogdir)
-
- reporoot = repo.origroot
- reponame = os.path.basename(reporoot)
- _removeoldlogfiles(userlogdir, reponame)
- logfile = _getlogfilename(logdir, username, reponame)
- except (OSError, IOError) as e:
- ui.debug('infinitepush backup log is disabled: %s\n' % e)
- except WrongPermissionsException as e:
- ui.debug(('%s directory has incorrect permission, ' +
- 'infinitepush backup logging will be disabled\n') %
- e.logdir)
- finally:
- os.umask(oldumask)
-
- if not logfile:
- logfile = os.devnull
-
- with open(logfile, 'a') as f:
- subprocess.Popen(background_cmd, shell=False, stdout=f,
- stderr=subprocess.STDOUT)
-
-def _dobackupcheck(bkpstate, ui, repo, dest, **opts):
- remotehexnodes = sorted(
- set(bkpstate.heads).union(bkpstate.localbookmarks.values()))
- if not remotehexnodes:
- return True
- other = _getremote(repo, ui, dest, **opts)
- batch = other.iterbatch()
- for hexnode in remotehexnodes:
- batch.lookup(hexnode)
- batch.submit()
- lookupresults = batch.results()
- i = 0
- try:
- for i, r in enumerate(lookupresults):
- # iterate over results to make it throw if revision
- # was not found
- pass
- return True
- except error.RepoError:
- ui.warn(_('unknown revision %r\n') % remotehexnodes[i])
- return False
-
-_backuplatestinfofile = 'infinitepushlatestbackupinfo'
-_backupstatefile = 'infinitepushbackupstate'
-_backupgenerationfile = 'infinitepushbackupgeneration'
-
-# Common helper functions
-def _getlocalinfo(repo):
- localinfo = {}
- localinfo['rev'] = repo[repo.changelog.tip()].rev()
- localinfo['time'] = int(time.time())
- return localinfo
-
-def _getlocalbookmarks(repo):
- localbookmarks = {}
- for bookmark, node in repo._bookmarks.iteritems():
- hexnode = hex(node)
- localbookmarks[bookmark] = hexnode
- return localbookmarks
-
-def _filterbookmarks(localbookmarks, repo, headstobackup):
- '''Filters out some bookmarks from being backed up
-
- Filters out bookmarks that do not point to ancestors of headstobackup or
- public commits
- '''
-
- headrevstobackup = [repo[hexhead].rev() for hexhead in headstobackup]
- ancestors = repo.changelog.ancestors(headrevstobackup, inclusive=True)
- filteredbooks = {}
- for bookmark, hexnode in localbookmarks.iteritems():
- if (repo[hexnode].rev() in ancestors or
- repo[hexnode].phase() == phases.public):
- filteredbooks[bookmark] = hexnode
- return filteredbooks
-
-def _downloadbackupstate(ui, other, sourcereporoot, sourcehostname, namingmgr):
- pattern = namingmgr.getcommonuserprefix()
- fetchedbookmarks = other.listkeyspatterns('bookmarks', patterns=[pattern])
- allbackupstates = collections.defaultdict(backupstate)
- for book, hexnode in fetchedbookmarks.iteritems():
- parsed = _parsebackupbookmark(book, namingmgr)
- if parsed:
- if sourcereporoot and sourcereporoot != parsed.reporoot:
- continue
- if sourcehostname and sourcehostname != parsed.hostname:
- continue
- key = (parsed.hostname, parsed.reporoot)
- if parsed.localbookmark:
- bookname = parsed.localbookmark
- allbackupstates[key].localbookmarks[bookname] = hexnode
- else:
- allbackupstates[key].heads.add(hexnode)
- else:
- ui.warn(_('wrong format of backup bookmark: %s') % book)
-
- return allbackupstates
-
-def _checkbackupstates(allbackupstates):
- if len(allbackupstates) == 0:
- raise error.Abort('no backups found!')
-
- hostnames = set(key[0] for key in allbackupstates.iterkeys())
- reporoots = set(key[1] for key in allbackupstates.iterkeys())
-
- if len(hostnames) > 1:
- raise error.Abort(
- _('ambiguous hostname to restore: %s') % sorted(hostnames),
- hint=_('set --hostname to disambiguate'))
-
- if len(reporoots) > 1:
- raise error.Abort(
- _('ambiguous repo root to restore: %s') % sorted(reporoots),
- hint=_('set --reporoot to disambiguate'))
-
-class BackupBookmarkNamingManager(object):
- def __init__(self, ui, repo, username=None):
- self.ui = ui
- self.repo = repo
- if not username:
- username = util.shortuser(ui.username())
- self.username = username
-
- self.hostname = self.ui.config('infinitepushbackup', 'hostname')
- if not self.hostname:
- self.hostname = socket.gethostname()
-
- def getcommonuserprefix(self):
- return '/'.join((self._getcommonuserprefix(), '*'))
-
- def getcommonprefix(self):
- return '/'.join((self._getcommonprefix(), '*'))
-
- def getbackupbookmarkprefix(self):
- return '/'.join((self._getbackupbookmarkprefix(), '*'))
-
- def getbackupbookmarkname(self, bookmark):
- bookmark = _escapebookmark(bookmark)
- return '/'.join((self._getbackupbookmarkprefix(), bookmark))
-
- def getbackupheadprefix(self):
- return '/'.join((self._getbackupheadprefix(), '*'))
-
- def getbackupheadname(self, hexhead):
- return '/'.join((self._getbackupheadprefix(), hexhead))
-
- def _getbackupbookmarkprefix(self):
- return '/'.join((self._getcommonprefix(), 'bookmarks'))
-
- def _getbackupheadprefix(self):
- return '/'.join((self._getcommonprefix(), 'heads'))
-
- def _getcommonuserprefix(self):
- return '/'.join(('infinitepush', 'backups', self.username))
-
- def _getcommonprefix(self):
- reporoot = self.repo.origroot
-
- result = '/'.join((self._getcommonuserprefix(), self.hostname))
- if not reporoot.startswith('/'):
- result += '/'
- result += reporoot
- if result.endswith('/'):
- result = result[:-1]
- return result
-
-def _escapebookmark(bookmark):
- '''
- If `bookmark` contains "bookmarks" as a substring then replace it with
- "bookmarksbookmarks". This will make parsing remote bookmark name
- unambigious.
- '''
-
- bookmark = encoding.fromlocal(bookmark)
- return bookmark.replace('bookmarks', 'bookmarksbookmarks')
-
-def _unescapebookmark(bookmark):
- bookmark = encoding.tolocal(bookmark)
- return bookmark.replace('bookmarksbookmarks', 'bookmarks')
-
-def _getremote(repo, ui, dest, **opts):
- path = ui.paths.getpath(dest, default=('infinitepush', 'default'))
- if not path:
- raise error.Abort(_('default repository not configured!'),
- hint=_("see 'hg help config.paths'"))
- dest = path.pushloc or path.loc
- return hg.peer(repo, opts, dest)
-
-def _getcommandandoptions(command):
- cmd = commands.table[command][0]
- opts = dict(opt[1:3] for opt in commands.table[command][1])
- return cmd, opts
-
-# Backup helper functions
-
-def _deltaparent(orig, self, revlog, rev, p1, p2, prev):
- # This version of deltaparent prefers p1 over prev to use less space
- dp = revlog.deltaparent(rev)
- if dp == nullrev and not revlog.storedeltachains:
- # send full snapshot only if revlog configured to do so
- return nullrev
- return p1
-
-def _getbookmarkstobackup(repo, newbookmarks, removedbookmarks,
- newheads, removedheads, namingmgr):
- bookmarkstobackup = {}
-
- for bookmark, hexnode in removedbookmarks.items():
- backupbookmark = namingmgr.getbackupbookmarkname(bookmark)
- bookmarkstobackup[backupbookmark] = ''
-
- for bookmark, hexnode in newbookmarks.items():
- backupbookmark = namingmgr.getbackupbookmarkname(bookmark)
- bookmarkstobackup[backupbookmark] = hexnode
-
- for hexhead in removedheads:
- headbookmarksname = namingmgr.getbackupheadname(hexhead)
- bookmarkstobackup[headbookmarksname] = ''
-
- for hexhead in newheads:
- headbookmarksname = namingmgr.getbackupheadname(hexhead)
- bookmarkstobackup[headbookmarksname] = hexhead
-
- return bookmarkstobackup
-
-def _createbundler(ui, repo, other):
- bundler = bundle2.bundle20(ui, bundle2.bundle2caps(other))
- # Disallow pushback because we want to avoid taking repo locks.
- # And we don't need pushback anyway
- capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo,
- allowpushback=False))
- bundler.newpart('replycaps', data=capsblob)
- return bundler
-
-def _sendbundle(bundler, other):
- stream = util.chunkbuffer(bundler.getchunks())
- try:
- other.unbundle(stream, ['force'], other.url())
- except error.BundleValueError as exc:
- raise error.Abort(_('missing support for %s') % exc)
-
-def findcommonoutgoing(repo, ui, other, heads):
- if heads:
- # Avoid using remotenames fastheaddiscovery heuristic. It uses
- # remotenames file to quickly find commonoutgoing set, but it can
- # result in sending public commits to infinitepush servers.
- # For example:
- #
- # o draft
- # /
- # o C1
- # |
- # ...
- # |
- # o remote/master
- #
- # pushbackup in that case results in sending to the infinitepush server
- # all public commits from 'remote/master' to C1. It increases size of
- # the bundle + it may result in storing data about public commits
- # in infinitepush table.
-
- with ui.configoverride({("remotenames", "fastheaddiscovery"): False}):
- nodes = map(repo.changelog.node, heads)
- return discovery.findcommonoutgoing(repo, other, onlyheads=nodes)
- else:
- return None
-
-def _getrevstobackup(repo, ui, other, headstobackup):
- # In rare cases it's possible to have a local node without filelogs.
- # This is possible if remotefilelog is enabled and if the node was
- # stripped server-side. We want to filter out these bad nodes and all
- # of their descendants.
- badnodes = ui.configlist('infinitepushbackup', 'dontbackupnodes', [])
- badnodes = [node for node in badnodes if node in repo]
- badrevs = [repo[node].rev() for node in badnodes]
- badnodesdescendants = repo.set('%ld::', badrevs) if badrevs else set()
- badnodesdescendants = set(ctx.hex() for ctx in badnodesdescendants)
- filteredheads = filter(lambda head: head in badnodesdescendants,
- headstobackup)
-
- if filteredheads:
- ui.warn(_('filtering nodes: %s\n') % filteredheads)
- ui.log('infinitepushbackup', 'corrupted nodes found',
- infinitepushbackupcorruptednodes='failure')
- headstobackup = filter(lambda head: head not in badnodesdescendants,
- headstobackup)
-
- revs = list(repo[hexnode].rev() for hexnode in headstobackup)
- outgoing = findcommonoutgoing(repo, ui, other, revs)
- nodeslimit = 1000
- if outgoing and len(outgoing.missing) > nodeslimit:
- # trying to push too many nodes usually means that there is a bug
- # somewhere. Let's be safe and avoid pushing too many nodes at once
- raise error.Abort('trying to back up too many nodes: %d' %
- (len(outgoing.missing),))
- return outgoing, set(filteredheads)
-
-def _localbackupstateexists(repo):
- return repo.vfs.exists(_backupstatefile)
-
-def _deletebackupstate(repo):
- return repo.vfs.tryunlink(_backupstatefile)
-
-def _readlocalbackupstate(ui, repo):
- repo = shareutil.getsrcrepo(repo)
- if not _localbackupstateexists(repo):
- return backupstate()
-
- with repo.vfs(_backupstatefile) as f:
- try:
- state = json.loads(f.read())
- if (not isinstance(state['bookmarks'], dict) or
- not isinstance(state['heads'], list)):
- raise ValueError('bad types of bookmarks or heads')
-
- result = backupstate()
- result.heads = set(map(str, state['heads']))
- result.localbookmarks = state['bookmarks']
- return result
- except (ValueError, KeyError, TypeError) as e:
- ui.warn(_('corrupt file: %s (%s)\n') % (_backupstatefile, e))
- return backupstate()
- return backupstate()
-
-def _writelocalbackupstate(vfs, heads, bookmarks):
- with vfs(_backupstatefile, 'w') as f:
- f.write(json.dumps({'heads': list(heads), 'bookmarks': bookmarks}))
-
-def _readbackupgenerationfile(vfs):
- try:
- with vfs(_backupgenerationfile) as f:
- return int(f.read())
- except (IOError, OSError, ValueError):
- return 0
-
-def _writebackupgenerationfile(vfs, backupgenerationvalue):
- with vfs(_backupgenerationfile, 'w', atomictemp=True) as f:
- f.write(str(backupgenerationvalue))
-
-def _writelocalbackupinfo(vfs, rev, time):
- with vfs(_backuplatestinfofile, 'w', atomictemp=True) as f:
- f.write(('backuprevision=%d\nbackuptime=%d\n') % (rev, time))
-
-# Restore helper functions
-def _parsebackupbookmark(backupbookmark, namingmgr):
- '''Parses backup bookmark and returns info about it
-
- Backup bookmark may represent either a local bookmark or a head.
- Returns None if backup bookmark has wrong format or tuple.
- First entry is a hostname where this bookmark came from.
- Second entry is a root of the repo where this bookmark came from.
- Third entry in a tuple is local bookmark if backup bookmark
- represents a local bookmark and None otherwise.
- '''
-
- backupbookmarkprefix = namingmgr._getcommonuserprefix()
- commonre = '^{0}/([-\w.]+)(/.*)'.format(re.escape(backupbookmarkprefix))
- bookmarkre = commonre + '/bookmarks/(.*)$'
- headsre = commonre + '/heads/[a-f0-9]{40}$'
-
- match = re.search(bookmarkre, backupbookmark)
- if not match:
- match = re.search(headsre, backupbookmark)
- if not match:
- return None
- # It's a local head not a local bookmark.
- # That's why localbookmark is None
- return backupbookmarktuple(hostname=match.group(1),
- reporoot=match.group(2),
- localbookmark=None)
-
- return backupbookmarktuple(hostname=match.group(1),
- reporoot=match.group(2),
- localbookmark=_unescapebookmark(match.group(3)))
-
-_timeformat = '%Y%m%d'
-
-def _getlogfilename(logdir, username, reponame):
- '''Returns name of the log file for particular user and repo
-
- Different users have different directories inside logdir. Log filename
- consists of reponame (basename of repo path) and current day
- (see _timeformat). That means that two different repos with the same name
- can share the same log file. This is not a big problem so we ignore it.
- '''
-
- currentday = time.strftime(_timeformat)
- return os.path.join(logdir, username, reponame + currentday)
-
-def _removeoldlogfiles(userlogdir, reponame):
- existinglogfiles = []
- for entry in osutil.listdir(userlogdir):
- filename = entry[0]
- fullpath = os.path.join(userlogdir, filename)
- if filename.startswith(reponame) and os.path.isfile(fullpath):
- try:
- time.strptime(filename[len(reponame):], _timeformat)
- except ValueError:
- continue
- existinglogfiles.append(filename)
-
- # _timeformat gives us a property that if we sort log file names in
- # descending order then newer files are going to be in the beginning
- existinglogfiles = sorted(existinglogfiles, reverse=True)
- # Delete logs that are older than 5 days
- maxlogfilenumber = 5
- if len(existinglogfiles) > maxlogfilenumber:
- for filename in existinglogfiles[maxlogfilenumber:]:
- os.unlink(os.path.join(userlogdir, filename))
-
-def _checkcommonlogdir(logdir):
- '''Checks permissions of the log directory
-
- We want log directory to actually be a directory, have restricting
- deletion flag set (sticky bit)
- '''
-
- try:
- st = os.stat(logdir)
- return stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX
- except OSError:
- # is raised by os.stat()
- return False
-
-def _checkuserlogdir(userlogdir):
- '''Checks permissions of the user log directory
-
- We want user log directory to be writable only by the user who created it
- and be owned by `username`
- '''
-
- try:
- st = os.stat(userlogdir)
- # Check that `userlogdir` is owned by `username`
- if os.getuid() != st.st_uid:
- return False
- return ((st.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)) ==
- stat.S_IWUSR)
- except OSError:
- # is raised by os.stat()
- return False
-
-def _dictdiff(first, second):
- '''Returns new dict that contains items from the first dict that are missing
- from the second dict.
- '''
- result = {}
- for book, hexnode in first.items():
- if second.get(book) != hexnode:
- result[book] = hexnode
- return result