comparison hgext/largefiles/remotestore.py @ 17127:9e1616307c4c

largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)

This implements a part of issue 3386.  It batches the request for the status of
all largefiles in the revisions that are about to be pushed into a single
request, instead of doing N separate requests.

In a real world test case, this change was verified to save 1,116 round-trips to
the server.  It only requires a client-side change; it is backwards-compatible
with an older version of the server.

17126:8fa8717b47b6

import urllib2

from mercurial import util
from mercurial.i18n import _

import lfutil
import basestore

class remotestore(basestore.basestore):
    '''a largefile store accessed over a network'''
    def __init__(self, ui, repo, url):
        super(remotestore, self).__init__(ui, repo, url)

    def put(self, source, hash):
        if self._verify(hash):
            return
        if self.sendfile(source, hash):
            raise util.Abort(
                _('remotestore: could not put %s to remote store %s')
                % (source, self.url))
        self.ui.debug(
            _('remotestore: put %s to remote store %s') % (source, self.url))

    def exists(self, hash):
        return self._verify(hash)

    def sendfile(self, filename, hash):
        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
        fd = None
        try:

        # Mercurial does not close its SSH connections after writing a stream
        if length is not None:
            infile = lfutil.limitreader(infile, length)
        return lfutil.copyandhash(lfutil.blockstream(infile), tmpfile)

    def _verify(self, hash):
        return not self._stat(hash)

    def _verifyfile(self, cctx, cset, contents, standin, verified):
        filename = lfutil.splitstandin(standin)
        if not filename:
            return False

                % (cset, filename))
            return True # failed
        else:
            raise RuntimeError('verify failed: unexpected response from '
                               'statlfile (%r)' % stat)

17127:9e1616307c4c

import urllib2

from mercurial import util
from mercurial.i18n import _
from mercurial.wireproto import remotebatch

import lfutil
import basestore

class remotestore(basestore.basestore):
    '''a largefile store accessed over a network'''
    def __init__(self, ui, repo, url):
        super(remotestore, self).__init__(ui, repo, url)

    def put(self, source, hash):
        if self.sendfile(source, hash):
            raise util.Abort(
                _('remotestore: could not put %s to remote store %s')
                % (source, self.url))
        self.ui.debug(
            _('remotestore: put %s to remote store %s') % (source, self.url))

    def exists(self, hashes):
        return self._verify(hashes)

    def sendfile(self, filename, hash):
        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
        fd = None
        try:

        # Mercurial does not close its SSH connections after writing a stream
        if length is not None:
            infile = lfutil.limitreader(infile, length)
        return lfutil.copyandhash(lfutil.blockstream(infile), tmpfile)

    def _verify(self, hashes):
        return self._stat(hashes)

    def _verifyfile(self, cctx, cset, contents, standin, verified):
        filename = lfutil.splitstandin(standin)
        if not filename:
            return False

                % (cset, filename))
            return True # failed
        else:
            raise RuntimeError('verify failed: unexpected response from '
                               'statlfile (%r)' % stat)

    def batch(self):
        '''Support for remote batching.'''
        return remotebatch(self)