hgext/largefiles/localstore.py
author Raphaël Gomès <rgomes@octobus.net>
Mon, 23 Mar 2020 12:11:41 +0100
changeset 44599 d31d1c0685be
parent 43085 eef9a2d67051
child 45942 89a2afe31e82
permissions -rw-r--r--
rust: update all dependencies We do this periodically to say up to date. No major versions were crossed this time per se, but the `rand` is still in v0, and their 0.7x series broke three things: - Some distribution-related elements were moved to a separate crate, flashing a deprecation warning - The `LogNormal::new` associated function now returns a `Result` - Certain RNGs were updated to sample a `u32` instead of `usize` when their upper-bound is less than `u32::MAX` for better portability, which changed the output for 2 tests. Moreover, the recent use of the `regex` crate for ignore mechanisms prompted some benchmarking that revealed that `regex` was slower at compiling big regex than `Re2`. The author of `regex` was very quick to discover an optimization that yielded a 30% improvement. It's still slower than `Re2` in that regard, but less so in the 1.3.6 release. Differential Revision: https://phab.mercurial-scm.org/D8320
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2009-2010 Gregory P. Ward
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
# Copyright 2009-2010 Intelerad Medical Systems Incorporated
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
# Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     7
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15168
diff changeset
     9
'''store class for local filesystem'''
29310
f89f83c8393a py3: make largefiles/localstore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    10
from __future__ import absolute_import
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
from mercurial.i18n import _
43085
eef9a2d67051 py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
    13
from mercurial.pycompat import open
30180
736f92c44656 largefiles: always use filechunkiter when iterating files
Mads Kiilerich <madski@unity3d.com>
parents: 29421
diff changeset
    14
from mercurial import util
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    15
29310
f89f83c8393a py3: make largefiles/localstore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    16
from . import (
f89f83c8393a py3: make largefiles/localstore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    17
    basestore,
f89f83c8393a py3: make largefiles/localstore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    18
    lfutil,
f89f83c8393a py3: make largefiles/localstore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    19
)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    20
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    21
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    22
class localstore(basestore.basestore):
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    23
    '''localstore first attempts to grab files out of the store in the remote
17424
e7cfe3587ea4 fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents: 17191
diff changeset
    24
    Mercurial repository.  Failing that, it attempts to grab the files from
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    25
    the user cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    27
    def __init__(self, ui, repo, remote):
17191
5884812686f7 peer: introduce peer methods to prepare for peer classes
Sune Foldager <cryo@cyanite.org>
parents: 16928
diff changeset
    28
        self.remote = remote.local()
18155
5206af8894a3 largefiles: cleanup of warnings on errors getting largefiles
Mads Kiilerich <madski@unity3d.com>
parents: 17439
diff changeset
    29
        super(localstore, self).__init__(ui, repo, self.remote.url())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    30
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    31
    def put(self, source, hash):
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    32
        if lfutil.instore(self.remote, hash):
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    33
            return
19007
266b5fb72f26 largefiles: 'put' should store 'source' file in under 'hash', also in localstore
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
    34
        lfutil.link(source, lfutil.storepath(self.remote, hash))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    35
17411
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    36
    def exists(self, hashes):
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    37
        retval = {}
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    38
        for hash in hashes:
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    39
            retval[hash] = lfutil.instore(self.remote, hash)
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    40
        return retval
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
    41
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    42
    def _getfile(self, tmpfile, filename, hash):
19000
eaf146e811a4 largefiles: refactoring - use findfile in localstore._getfile
Mads Kiilerich <madski@unity3d.com>
parents: 18998
diff changeset
    43
        path = lfutil.findfile(self.remote, hash)
eaf146e811a4 largefiles: refactoring - use findfile in localstore._getfile
Mads Kiilerich <madski@unity3d.com>
parents: 18998
diff changeset
    44
        if not path:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    45
            raise basestore.StoreError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    46
                filename, hash, self.url, _(b"can't get file locally")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    47
            )
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    48
        with open(path, b'rb') as fd:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    49
            return lfutil.copyandhash(util.filechunkiter(fd), tmpfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    50
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    51
    def _verifyfiles(self, contents, filestocheck):
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    52
        failed = False
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    53
        for cset, filename, expectedhash in filestocheck:
29421
ecbbf4d56ee8 largefiles: check file in the repo store before checking remotely (issue5257)
liscju <piotr.listkiewicz@gmail.com>
parents: 29409
diff changeset
    54
            storepath, exists = lfutil.findstorepath(self.repo, expectedhash)
ecbbf4d56ee8 largefiles: check file in the repo store before checking remotely (issue5257)
liscju <piotr.listkiewicz@gmail.com>
parents: 29409
diff changeset
    55
            if not exists:
ecbbf4d56ee8 largefiles: check file in the repo store before checking remotely (issue5257)
liscju <piotr.listkiewicz@gmail.com>
parents: 29409
diff changeset
    56
                storepath, exists = lfutil.findstorepath(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    57
                    self.remote, expectedhash
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    58
                )
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    59
            if not exists:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    60
                self.ui.warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    61
                    _(b'changeset %s: %s references missing %s\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    62
                    % (cset, filename, storepath)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    63
                )
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    64
                failed = True
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    65
            elif contents:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    66
                actualhash = lfutil.hashfile(storepath)
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    67
                if actualhash != expectedhash:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    68
                    self.ui.warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    69
                        _(b'changeset %s: %s references corrupted %s\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    70
                        % (cset, filename, storepath)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 30180
diff changeset
    71
                    )
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    72
                    failed = True
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 27769
diff changeset
    73
        return failed