hgext/remotefilelog/basestore.py

43075:57875cf423c9

)
from . import (
    constants,
    shallowutil,
)

class basestore(object):
    def __init__(self, repo, path, reponame, shared=False):
        """Creates a remotefilelog store object for the given repo name.

        self._path = path
        self._reponame = reponame
        self._shared = shared
        self._uid = os.getuid() if not pycompat.iswindows else None

        self._validatecachelog = self.ui.config("remotefilelog",
                                                "validatecachelog")
        self._validatecache = self.ui.config("remotefilelog", "validatecache",
                                             'on')
        if self._validatecache not in ('on', 'strict', 'off'):
            self._validatecache = 'on'
        if self._validatecache == 'off':
            self._validatecache = False

    def getmissing(self, keys):
        missing = []
        for name, node in keys:
            filepath = self._getfilepath(name, node)
            exists = os.path.exists(filepath)
            if (exists and self._validatecache == 'strict' and
                not self._validatekey(filepath, 'contains')):
                exists = False
            if not exists:
                missing.append((name, node))

        return missing

    def cleanup(self, ledger):
        ui = self.ui
        entries = ledger.sources.get(self, [])
        count = 0
        progress = ui.makeprogress(_("cleaning up"), unit="files",
                                   total=len(entries))
        for entry in entries:
            if entry.gced or (entry.datarepacked and entry.historyrepacked):
                progress.update(count)
                path = self._getfilepath(entry.filename, entry.node)
                util.tryunlink(path)

                    missingfilename.discard(sha)

        return filenames

    def _getrepocachepath(self):
        return os.path.join(
            self._path, self._reponame) if self._shared else self._path

    def _listkeys(self):
        """List all the remotefilelog keys that exist in the store.

        Returns a iterator of (filename hash, filecontent hash) tuples.

                    with open(self._validatecachelog, 'a+') as f:
                        f.write("corrupt %s during read\n" % filepath)
                os.rename(filepath, filepath + ".corrupt")
                raise KeyError("corrupt local cache file %s" % filepath)
        except IOError:
            raise KeyError("no file found at %s for %s:%s" % (filepath, name,
                                                              hex(node)))

        return data

    def addremotefilelognode(self, name, node, data):
        filepath = self._getfilepath(name, node)

            shallowutil.mkstickygroupdir(self.ui, os.path.dirname(filepath))
            shallowutil.writefile(filepath, data, readonly=True)

            if self._validatecache:
                if not self._validatekey(filepath, 'write'):
                    raise error.Abort(_("local cache write was corrupted %s") %
                                      filepath)
        finally:
            os.umask(oldumask)

    def markrepo(self, path):
        """Call this to add the given repo path to the store's list of

                    # it is truncated
                    return False

                # extract the node from the metadata
                offset += size
                datanode = data[offset:offset + 20]

                # and compare against the path
                if os.path.basename(path) == hex(datanode):
                    # Content matches the intended path
                    return True

        removed = 0

        # keep files newer than a day even if they aren't needed
        limit = time.time() - (60 * 60 * 24)

        progress = ui.makeprogress(_("removing unnecessary files"),
                                   unit="files")
        progress.update(0)
        for root, dirs, files in os.walk(cachepath):
            for file in files:
                if file == 'repos':
                    continue

                        shallowutil.unlinkfile(path)
                    except OSError as e:
                        # errno.ENOENT = no such file or directory
                        if e.errno != errno.ENOENT:
                            raise
                        msg = _("warning: file %s was removed by another "
                                "process\n")
                        ui.warn(msg % path)
                        continue
                    removed += 1
        progress.complete()

        # remove oldest files until under limit
        limit = ui.configbytes("remotefilelog", "cachelimit")
        if size > limit:
            excess = size - limit
            progress = ui.makeprogress(_("enforcing cache limit"), unit="bytes",
                                       total=excess)
            removedexcess = 0
            while queue and size > limit and size > 0:
                progress.update(removedexcess)
                atime, oldpath, oldpathstat = queue.get()
                try:

                size -= oldpathstat.st_size
                removed += 1
                removedexcess += oldpathstat.st_size
            progress.complete()

        ui.status(_("finished: removed %d of %d files (%0.2f GB to %0.2f GB)\n")
                  % (removed, count,
                     float(originalsize) / 1024.0 / 1024.0 / 1024.0,
                     float(size) / 1024.0 / 1024.0 / 1024.0))

class baseunionstore(object):
    def __init__(self, *args, **kwargs):
        # If one of the functions that iterates all of the stores is about to
        # throw a KeyError, try this many times with a full refresh between

    @staticmethod
    def retriable(fn):
        def noop(*args):
            pass
        def wrapped(self, *args, **kwargs):
            retrylog = self.retrylog or noop
            funcname = fn.__name__
            i = 0
            while i < self.numattempts:

                try:
                    return fn(self, *args, **kwargs)
                except KeyError:
                    if i == self.numattempts:
                        # retries exhausted
                        retrylog('retries exhausted in %s, raising KeyError\n' %
                                 pycompat.sysbytes(funcname))
                        raise
        return wrapped

43076:2372284d9457

)
from . import (
    constants,
    shallowutil,
)


class basestore(object):
    def __init__(self, repo, path, reponame, shared=False):
        """Creates a remotefilelog store object for the given repo name.

        self._path = path
        self._reponame = reponame
        self._shared = shared
        self._uid = os.getuid() if not pycompat.iswindows else None

        self._validatecachelog = self.ui.config(
            "remotefilelog", "validatecachelog"
        )
        self._validatecache = self.ui.config(
            "remotefilelog", "validatecache", 'on'
        )
        if self._validatecache not in ('on', 'strict', 'off'):
            self._validatecache = 'on'
        if self._validatecache == 'off':
            self._validatecache = False

    def getmissing(self, keys):
        missing = []
        for name, node in keys:
            filepath = self._getfilepath(name, node)
            exists = os.path.exists(filepath)
            if (
                exists
                and self._validatecache == 'strict'
                and not self._validatekey(filepath, 'contains')
            ):
                exists = False
            if not exists:
                missing.append((name, node))

        return missing

    def cleanup(self, ledger):
        ui = self.ui
        entries = ledger.sources.get(self, [])
        count = 0
        progress = ui.makeprogress(
            _("cleaning up"), unit="files", total=len(entries)
        )
        for entry in entries:
            if entry.gced or (entry.datarepacked and entry.historyrepacked):
                progress.update(count)
                path = self._getfilepath(entry.filename, entry.node)
                util.tryunlink(path)

                    missingfilename.discard(sha)

        return filenames

    def _getrepocachepath(self):
        return (
            os.path.join(self._path, self._reponame)
            if self._shared
            else self._path
        )

    def _listkeys(self):
        """List all the remotefilelog keys that exist in the store.

        Returns a iterator of (filename hash, filecontent hash) tuples.

                    with open(self._validatecachelog, 'a+') as f:
                        f.write("corrupt %s during read\n" % filepath)
                os.rename(filepath, filepath + ".corrupt")
                raise KeyError("corrupt local cache file %s" % filepath)
        except IOError:
            raise KeyError(
                "no file found at %s for %s:%s" % (filepath, name, hex(node))
            )

        return data

    def addremotefilelognode(self, name, node, data):
        filepath = self._getfilepath(name, node)

            shallowutil.mkstickygroupdir(self.ui, os.path.dirname(filepath))
            shallowutil.writefile(filepath, data, readonly=True)

            if self._validatecache:
                if not self._validatekey(filepath, 'write'):
                    raise error.Abort(
                        _("local cache write was corrupted %s") % filepath
                    )
        finally:
            os.umask(oldumask)

    def markrepo(self, path):
        """Call this to add the given repo path to the store's list of

                    # it is truncated
                    return False

                # extract the node from the metadata
                offset += size
                datanode = data[offset : offset + 20]

                # and compare against the path
                if os.path.basename(path) == hex(datanode):
                    # Content matches the intended path
                    return True

        removed = 0

        # keep files newer than a day even if they aren't needed
        limit = time.time() - (60 * 60 * 24)

        progress = ui.makeprogress(
            _("removing unnecessary files"), unit="files"
        )
        progress.update(0)
        for root, dirs, files in os.walk(cachepath):
            for file in files:
                if file == 'repos':
                    continue

                        shallowutil.unlinkfile(path)
                    except OSError as e:
                        # errno.ENOENT = no such file or directory
                        if e.errno != errno.ENOENT:
                            raise
                        msg = _(
                            "warning: file %s was removed by another "
                            "process\n"
                        )
                        ui.warn(msg % path)
                        continue
                    removed += 1
        progress.complete()

        # remove oldest files until under limit
        limit = ui.configbytes("remotefilelog", "cachelimit")
        if size > limit:
            excess = size - limit
            progress = ui.makeprogress(
                _("enforcing cache limit"), unit="bytes", total=excess
            )
            removedexcess = 0
            while queue and size > limit and size > 0:
                progress.update(removedexcess)
                atime, oldpath, oldpathstat = queue.get()
                try:

                size -= oldpathstat.st_size
                removed += 1
                removedexcess += oldpathstat.st_size
            progress.complete()

        ui.status(
            _("finished: removed %d of %d files (%0.2f GB to %0.2f GB)\n")
            % (
                removed,
                count,
                float(originalsize) / 1024.0 / 1024.0 / 1024.0,
                float(size) / 1024.0 / 1024.0 / 1024.0,
            )
        )


class baseunionstore(object):
    def __init__(self, *args, **kwargs):
        # If one of the functions that iterates all of the stores is about to
        # throw a KeyError, try this many times with a full refresh between

    @staticmethod
    def retriable(fn):
        def noop(*args):
            pass

        def wrapped(self, *args, **kwargs):
            retrylog = self.retrylog or noop
            funcname = fn.__name__
            i = 0
            while i < self.numattempts:

                try:
                    return fn(self, *args, **kwargs)
                except KeyError:
                    if i == self.numattempts:
                        # retries exhausted
                        retrylog(
                            'retries exhausted in %s, raising KeyError\n'
                            % pycompat.sysbytes(funcname)
                        )
                        raise

        return wrapped