changeset 31217:0f31830fbfc4

vfs: extract 'vfs' class and related code to a new 'vfs' module (API) The 'scmutil' is growing large (1500+ lines) and 2/5 of it is related to vfs. We extract the 'vfs' related code in its own module get both module back to a better scale and clearer contents. We keep all the references available in 'scmutil' for now as many reference needs to be updated.
author Pierre-Yves David <pierre-yves.david@ens-lyon.org>
date Wed, 01 Mar 2017 11:00:12 +0100
parents 21fa3d3688f3
children 4cc3797aa59c
files mercurial/scmutil.py mercurial/vfs.py
diffstat 2 files changed, 647 insertions(+), 616 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial/scmutil.py	Thu Mar 02 03:52:36 2017 +0100
+++ b/mercurial/scmutil.py	Wed Mar 01 11:00:12 2017 +0100
@@ -7,17 +7,12 @@
 
 from __future__ import absolute_import
 
-import contextlib
 import errno
 import glob
 import hashlib
 import os
 import re
-import shutil
 import socket
-import stat
-import tempfile
-import threading
 
 from .i18n import _
 from .node import wdirrev
@@ -32,6 +27,7 @@
     revsetlang,
     similar,
     util,
+    vfs as vfsmod,
 )
 
 if pycompat.osname == 'nt':
@@ -336,455 +332,16 @@
         key = s.digest()
     return key
 
-class abstractvfs(object):
-    """Abstract base class; cannot be instantiated"""
-
-    def __init__(self, *args, **kwargs):
-        '''Prevent instantiation; don't call this from subclasses.'''
-        raise NotImplementedError('attempted instantiating ' + str(type(self)))
-
-    def tryread(self, path):
-        '''gracefully return an empty string for missing files'''
-        try:
-            return self.read(path)
-        except IOError as inst:
-            if inst.errno != errno.ENOENT:
-                raise
-        return ""
-
-    def tryreadlines(self, path, mode='rb'):
-        '''gracefully return an empty array for missing files'''
-        try:
-            return self.readlines(path, mode=mode)
-        except IOError as inst:
-            if inst.errno != errno.ENOENT:
-                raise
-        return []
-
-    @util.propertycache
-    def open(self):
-        '''Open ``path`` file, which is relative to vfs root.
-
-        Newly created directories are marked as "not to be indexed by
-        the content indexing service", if ``notindexed`` is specified
-        for "write" mode access.
-        '''
-        return self.__call__
-
-    def read(self, path):
-        with self(path, 'rb') as fp:
-            return fp.read()
-
-    def readlines(self, path, mode='rb'):
-        with self(path, mode=mode) as fp:
-            return fp.readlines()
-
-    def write(self, path, data, backgroundclose=False):
-        with self(path, 'wb', backgroundclose=backgroundclose) as fp:
-            return fp.write(data)
-
-    def writelines(self, path, data, mode='wb', notindexed=False):
-        with self(path, mode=mode, notindexed=notindexed) as fp:
-            return fp.writelines(data)
-
-    def append(self, path, data):
-        with self(path, 'ab') as fp:
-            return fp.write(data)
-
-    def basename(self, path):
-        """return base element of a path (as os.path.basename would do)
-
-        This exists to allow handling of strange encoding if needed."""
-        return os.path.basename(path)
-
-    def chmod(self, path, mode):
-        return os.chmod(self.join(path), mode)
-
-    def dirname(self, path):
-        """return dirname element of a path (as os.path.dirname would do)
-
-        This exists to allow handling of strange encoding if needed."""
-        return os.path.dirname(path)
-
-    def exists(self, path=None):
-        return os.path.exists(self.join(path))
-
-    def fstat(self, fp):
-        return util.fstat(fp)
-
-    def isdir(self, path=None):
-        return os.path.isdir(self.join(path))
-
-    def isfile(self, path=None):
-        return os.path.isfile(self.join(path))
-
-    def islink(self, path=None):
-        return os.path.islink(self.join(path))
-
-    def isfileorlink(self, path=None):
-        '''return whether path is a regular file or a symlink
-
-        Unlike isfile, this doesn't follow symlinks.'''
-        try:
-            st = self.lstat(path)
-        except OSError:
-            return False
-        mode = st.st_mode
-        return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
-
-    def reljoin(self, *paths):
-        """join various elements of a path together (as os.path.join would do)
-
-        The vfs base is not injected so that path stay relative. This exists
-        to allow handling of strange encoding if needed."""
-        return os.path.join(*paths)
-
-    def split(self, path):
-        """split top-most element of a path (as os.path.split would do)
-
-        This exists to allow handling of strange encoding if needed."""
-        return os.path.split(path)
-
-    def lexists(self, path=None):
-        return os.path.lexists(self.join(path))
-
-    def lstat(self, path=None):
-        return os.lstat(self.join(path))
-
-    def listdir(self, path=None):
-        return os.listdir(self.join(path))
-
-    def makedir(self, path=None, notindexed=True):
-        return util.makedir(self.join(path), notindexed)
-
-    def makedirs(self, path=None, mode=None):
-        return util.makedirs(self.join(path), mode)
-
-    def makelock(self, info, path):
-        return util.makelock(info, self.join(path))
-
-    def mkdir(self, path=None):
-        return os.mkdir(self.join(path))
-
-    def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
-        fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
-                                    dir=self.join(dir), text=text)
-        dname, fname = util.split(name)
-        if dir:
-            return fd, os.path.join(dir, fname)
-        else:
-            return fd, fname
-
-    def readdir(self, path=None, stat=None, skip=None):
-        return osutil.listdir(self.join(path), stat, skip)
-
-    def readlock(self, path):
-        return util.readlock(self.join(path))
-
-    def rename(self, src, dst, checkambig=False):
-        """Rename from src to dst
-
-        checkambig argument is used with util.filestat, and is useful
-        only if destination file is guarded by any lock
-        (e.g. repo.lock or repo.wlock).
-        """
-        dstpath = self.join(dst)
-        oldstat = checkambig and util.filestat(dstpath)
-        if oldstat and oldstat.stat:
-            ret = util.rename(self.join(src), dstpath)
-            newstat = util.filestat(dstpath)
-            if newstat.isambig(oldstat):
-                # stat of renamed file is ambiguous to original one
-                newstat.avoidambig(dstpath, oldstat)
-            return ret
-        return util.rename(self.join(src), dstpath)
-
-    def readlink(self, path):
-        return os.readlink(self.join(path))
-
-    def removedirs(self, path=None):
-        """Remove a leaf directory and all empty intermediate ones
-        """
-        return util.removedirs(self.join(path))
-
-    def rmtree(self, path=None, ignore_errors=False, forcibly=False):
-        """Remove a directory tree recursively
-
-        If ``forcibly``, this tries to remove READ-ONLY files, too.
-        """
-        if forcibly:
-            def onerror(function, path, excinfo):
-                if function is not os.remove:
-                    raise
-                # read-only files cannot be unlinked under Windows
-                s = os.stat(path)
-                if (s.st_mode & stat.S_IWRITE) != 0:
-                    raise
-                os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
-                os.remove(path)
-        else:
-            onerror = None
-        return shutil.rmtree(self.join(path),
-                             ignore_errors=ignore_errors, onerror=onerror)
-
-    def setflags(self, path, l, x):
-        return util.setflags(self.join(path), l, x)
-
-    def stat(self, path=None):
-        return os.stat(self.join(path))
-
-    def unlink(self, path=None):
-        return util.unlink(self.join(path))
-
-    def unlinkpath(self, path=None, ignoremissing=False):
-        return util.unlinkpath(self.join(path), ignoremissing)
-
-    def utime(self, path=None, t=None):
-        return os.utime(self.join(path), t)
-
-    def walk(self, path=None, onerror=None):
-        """Yield (dirpath, dirs, files) tuple for each directories under path
-
-        ``dirpath`` is relative one from the root of this vfs. This
-        uses ``os.sep`` as path separator, even you specify POSIX
-        style ``path``.
-
-        "The root of this vfs" is represented as empty ``dirpath``.
-        """
-        root = os.path.normpath(self.join(None))
-        # when dirpath == root, dirpath[prefixlen:] becomes empty
-        # because len(dirpath) < prefixlen.
-        prefixlen = len(pathutil.normasprefix(root))
-        for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
-            yield (dirpath[prefixlen:], dirs, files)
-
-    @contextlib.contextmanager
-    def backgroundclosing(self, ui, expectedcount=-1):
-        """Allow files to be closed asynchronously.
-
-        When this context manager is active, ``backgroundclose`` can be passed
-        to ``__call__``/``open`` to result in the file possibly being closed
-        asynchronously, on a background thread.
-        """
-        # This is an arbitrary restriction and could be changed if we ever
-        # have a use case.
-        vfs = getattr(self, 'vfs', self)
-        if getattr(vfs, '_backgroundfilecloser', None):
-            raise error.Abort(
-                _('can only have 1 active background file closer'))
-
-        with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
-            try:
-                vfs._backgroundfilecloser = bfc
-                yield bfc
-            finally:
-                vfs._backgroundfilecloser = None
-
-class vfs(abstractvfs):
-    '''Operate files relative to a base directory
-
-    This class is used to hide the details of COW semantics and
-    remote file access from higher level code.
-    '''
-    def __init__(self, base, audit=True, expandpath=False, realpath=False):
-        if expandpath:
-            base = util.expandpath(base)
-        if realpath:
-            base = os.path.realpath(base)
-        self.base = base
-        self.mustaudit = audit
-        self.createmode = None
-        self._trustnlink = None
-
-    @property
-    def mustaudit(self):
-        return self._audit
-
-    @mustaudit.setter
-    def mustaudit(self, onoff):
-        self._audit = onoff
-        if onoff:
-            self.audit = pathutil.pathauditor(self.base)
-        else:
-            self.audit = util.always
-
-    @util.propertycache
-    def _cansymlink(self):
-        return util.checklink(self.base)
-
-    @util.propertycache
-    def _chmod(self):
-        return util.checkexec(self.base)
-
-    def _fixfilemode(self, name):
-        if self.createmode is None or not self._chmod:
-            return
-        os.chmod(name, self.createmode & 0o666)
-
-    def __call__(self, path, mode="r", text=False, atomictemp=False,
-                 notindexed=False, backgroundclose=False, checkambig=False):
-        '''Open ``path`` file, which is relative to vfs root.
-
-        Newly created directories are marked as "not to be indexed by
-        the content indexing service", if ``notindexed`` is specified
-        for "write" mode access.
-
-        If ``backgroundclose`` is passed, the file may be closed asynchronously.
-        It can only be used if the ``self.backgroundclosing()`` context manager
-        is active. This should only be specified if the following criteria hold:
-
-        1. There is a potential for writing thousands of files. Unless you
-           are writing thousands of files, the performance benefits of
-           asynchronously closing files is not realized.
-        2. Files are opened exactly once for the ``backgroundclosing``
-           active duration and are therefore free of race conditions between
-           closing a file on a background thread and reopening it. (If the
-           file were opened multiple times, there could be unflushed data
-           because the original file handle hasn't been flushed/closed yet.)
-
-        ``checkambig`` argument is passed to atomictemplfile (valid
-        only for writing), and is useful only if target file is
-        guarded by any lock (e.g. repo.lock or repo.wlock).
-        '''
-        if self._audit:
-            r = util.checkosfilename(path)
-            if r:
-                raise error.Abort("%s: %r" % (r, path))
-        self.audit(path)
-        f = self.join(path)
-
-        if not text and "b" not in mode:
-            mode += "b" # for that other OS
-
-        nlink = -1
-        if mode not in ('r', 'rb'):
-            dirname, basename = util.split(f)
-            # If basename is empty, then the path is malformed because it points
-            # to a directory. Let the posixfile() call below raise IOError.
-            if basename:
-                if atomictemp:
-                    util.makedirs(dirname, self.createmode, notindexed)
-                    return util.atomictempfile(f, mode, self.createmode,
-                                               checkambig=checkambig)
-                try:
-                    if 'w' in mode:
-                        util.unlink(f)
-                        nlink = 0
-                    else:
-                        # nlinks() may behave differently for files on Windows
-                        # shares if the file is open.
-                        with util.posixfile(f):
-                            nlink = util.nlinks(f)
-                            if nlink < 1:
-                                nlink = 2 # force mktempcopy (issue1922)
-                except (OSError, IOError) as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                    nlink = 0
-                    util.makedirs(dirname, self.createmode, notindexed)
-                if nlink > 0:
-                    if self._trustnlink is None:
-                        self._trustnlink = nlink > 1 or util.checknlink(f)
-                    if nlink > 1 or not self._trustnlink:
-                        util.rename(util.mktempcopy(f), f)
-        fp = util.posixfile(f, mode)
-        if nlink == 0:
-            self._fixfilemode(f)
-
-        if checkambig:
-            if mode in ('r', 'rb'):
-                raise error.Abort(_('implementation error: mode %s is not'
-                                    ' valid for checkambig=True') % mode)
-            fp = checkambigatclosing(fp)
-
-        if backgroundclose:
-            if not self._backgroundfilecloser:
-                raise error.Abort(_('backgroundclose can only be used when a '
-                                  'backgroundclosing context manager is active')
-                                  )
-
-            fp = delayclosedfile(fp, self._backgroundfilecloser)
-
-        return fp
-
-    def symlink(self, src, dst):
-        self.audit(dst)
-        linkname = self.join(dst)
-        try:
-            os.unlink(linkname)
-        except OSError:
-            pass
-
-        util.makedirs(os.path.dirname(linkname), self.createmode)
-
-        if self._cansymlink:
-            try:
-                os.symlink(src, linkname)
-            except OSError as err:
-                raise OSError(err.errno, _('could not symlink to %r: %s') %
-                              (src, err.strerror), linkname)
-        else:
-            self.write(dst, src)
-
-    def join(self, path, *insidef):
-        if path:
-            return os.path.join(self.base, path, *insidef)
-        else:
-            return self.base
-
-opener = vfs
-
-class auditvfs(object):
-    def __init__(self, vfs):
-        self.vfs = vfs
-
-    @property
-    def mustaudit(self):
-        return self.vfs.mustaudit
-
-    @mustaudit.setter
-    def mustaudit(self, onoff):
-        self.vfs.mustaudit = onoff
-
-    @property
-    def options(self):
-        return self.vfs.options
-
-    @options.setter
-    def options(self, value):
-        self.vfs.options = value
-
-class filtervfs(abstractvfs, auditvfs):
-    '''Wrapper vfs for filtering filenames with a function.'''
-
-    def __init__(self, vfs, filter):
-        auditvfs.__init__(self, vfs)
-        self._filter = filter
-
-    def __call__(self, path, *args, **kwargs):
-        return self.vfs(self._filter(path), *args, **kwargs)
-
-    def join(self, path, *insidef):
-        if path:
-            return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
-        else:
-            return self.vfs.join(path)
-
-filteropener = filtervfs
-
-class readonlyvfs(abstractvfs, auditvfs):
-    '''Wrapper vfs preventing any writing.'''
-
-    def __init__(self, vfs):
-        auditvfs.__init__(self, vfs)
-
-    def __call__(self, path, mode='r', *args, **kw):
-        if mode not in ('r', 'rb'):
-            raise error.Abort(_('this vfs is read only'))
-        return self.vfs(path, mode, *args, **kw)
-
-    def join(self, path, *insidef):
-        return self.vfs.join(path, *insidef)
+# compatibility layer since all 'vfs' code moved to 'mercurial.vfs'
+#
+# This is hard to instal deprecation warning to this since we do not have
+# access to a 'ui' object.
+opener = vfs = vfsmod.vfs
+filteropener = filtervfs = vfsmod.filtervfs
+abstractvfs = vfsmod.abstractvfs
+readonlyvfs = vfsmod.readonlyvfs
+auditvfs = vfsmod.auditvfs
+checkambigatclosing = vfsmod.checkambigatclosing
 
 def walkrepos(path, followsym=False, seen_dirs=None, recurse=False):
     '''yield every hg repository under path, always recursively.
@@ -1408,165 +965,3 @@
     """
     # experimental config: format.generaldelta
     return ui.configbool('format', 'generaldelta', False)
-
-class closewrapbase(object):
-    """Base class of wrapper, which hooks closing
-
-    Do not instantiate outside of the vfs layer.
-    """
-    def __init__(self, fh):
-        object.__setattr__(self, '_origfh', fh)
-
-    def __getattr__(self, attr):
-        return getattr(self._origfh, attr)
-
-    def __setattr__(self, attr, value):
-        return setattr(self._origfh, attr, value)
-
-    def __delattr__(self, attr):
-        return delattr(self._origfh, attr)
-
-    def __enter__(self):
-        return self._origfh.__enter__()
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        raise NotImplementedError('attempted instantiating ' + str(type(self)))
-
-    def close(self):
-        raise NotImplementedError('attempted instantiating ' + str(type(self)))
-
-class delayclosedfile(closewrapbase):
-    """Proxy for a file object whose close is delayed.
-
-    Do not instantiate outside of the vfs layer.
-    """
-    def __init__(self, fh, closer):
-        super(delayclosedfile, self).__init__(fh)
-        object.__setattr__(self, '_closer', closer)
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self._closer.close(self._origfh)
-
-    def close(self):
-        self._closer.close(self._origfh)
-
-class backgroundfilecloser(object):
-    """Coordinates background closing of file handles on multiple threads."""
-    def __init__(self, ui, expectedcount=-1):
-        self._running = False
-        self._entered = False
-        self._threads = []
-        self._threadexception = None
-
-        # Only Windows/NTFS has slow file closing. So only enable by default
-        # on that platform. But allow to be enabled elsewhere for testing.
-        defaultenabled = pycompat.osname == 'nt'
-        enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
-
-        if not enabled:
-            return
-
-        # There is overhead to starting and stopping the background threads.
-        # Don't do background processing unless the file count is large enough
-        # to justify it.
-        minfilecount = ui.configint('worker', 'backgroundcloseminfilecount',
-                                    2048)
-        # FUTURE dynamically start background threads after minfilecount closes.
-        # (We don't currently have any callers that don't know their file count)
-        if expectedcount > 0 and expectedcount < minfilecount:
-            return
-
-        # Windows defaults to a limit of 512 open files. A buffer of 128
-        # should give us enough headway.
-        maxqueue = ui.configint('worker', 'backgroundclosemaxqueue', 384)
-        threadcount = ui.configint('worker', 'backgroundclosethreadcount', 4)
-
-        ui.debug('starting %d threads for background file closing\n' %
-                 threadcount)
-
-        self._queue = util.queue(maxsize=maxqueue)
-        self._running = True
-
-        for i in range(threadcount):
-            t = threading.Thread(target=self._worker, name='backgroundcloser')
-            self._threads.append(t)
-            t.start()
-
-    def __enter__(self):
-        self._entered = True
-        return self
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self._running = False
-
-        # Wait for threads to finish closing so open files don't linger for
-        # longer than lifetime of context manager.
-        for t in self._threads:
-            t.join()
-
-    def _worker(self):
-        """Main routine for worker thread."""
-        while True:
-            try:
-                fh = self._queue.get(block=True, timeout=0.100)
-                # Need to catch or the thread will terminate and
-                # we could orphan file descriptors.
-                try:
-                    fh.close()
-                except Exception as e:
-                    # Stash so can re-raise from main thread later.
-                    self._threadexception = e
-            except util.empty:
-                if not self._running:
-                    break
-
-    def close(self, fh):
-        """Schedule a file for closing."""
-        if not self._entered:
-            raise error.Abort(_('can only call close() when context manager '
-                              'active'))
-
-        # If a background thread encountered an exception, raise now so we fail
-        # fast. Otherwise we may potentially go on for minutes until the error
-        # is acted on.
-        if self._threadexception:
-            e = self._threadexception
-            self._threadexception = None
-            raise e
-
-        # If we're not actively running, close synchronously.
-        if not self._running:
-            fh.close()
-            return
-
-        self._queue.put(fh, block=True, timeout=None)
-
-class checkambigatclosing(closewrapbase):
-    """Proxy for a file object, to avoid ambiguity of file stat
-
-    See also util.filestat for detail about "ambiguity of file stat".
-
-    This proxy is useful only if the target file is guarded by any
-    lock (e.g. repo.lock or repo.wlock)
-
-    Do not instantiate outside of the vfs layer.
-    """
-    def __init__(self, fh):
-        super(checkambigatclosing, self).__init__(fh)
-        object.__setattr__(self, '_oldstat', util.filestat(fh.name))
-
-    def _checkambig(self):
-        oldstat = self._oldstat
-        if oldstat.stat:
-            newstat = util.filestat(self._origfh.name)
-            if newstat.isambig(oldstat):
-                # stat of changed file is ambiguous to original one
-                newstat.avoidambig(self._origfh.name, oldstat)
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self._origfh.__exit__(exc_type, exc_value, exc_tb)
-        self._checkambig()
-
-    def close(self):
-        self._origfh.close()
-        self._checkambig()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/vfs.py	Wed Mar 01 11:00:12 2017 +0100
@@ -0,0 +1,636 @@
+# vfs.py - Mercurial 'vfs' classes
+#
+#  Copyright Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import os
+import shutil
+import stat
+import tempfile
+import threading
+
+from .i18n import _
+from . import (
+    error,
+    osutil,
+    pathutil,
+    pycompat,
+    util,
+)
+
+class abstractvfs(object):
+    """Abstract base class; cannot be instantiated"""
+
+    def __init__(self, *args, **kwargs):
+        '''Prevent instantiation; don't call this from subclasses.'''
+        raise NotImplementedError('attempted instantiating ' + str(type(self)))
+
+    def tryread(self, path):
+        '''gracefully return an empty string for missing files'''
+        try:
+            return self.read(path)
+        except IOError as inst:
+            if inst.errno != errno.ENOENT:
+                raise
+        return ""
+
+    def tryreadlines(self, path, mode='rb'):
+        '''gracefully return an empty array for missing files'''
+        try:
+            return self.readlines(path, mode=mode)
+        except IOError as inst:
+            if inst.errno != errno.ENOENT:
+                raise
+        return []
+
+    @util.propertycache
+    def open(self):
+        '''Open ``path`` file, which is relative to vfs root.
+
+        Newly created directories are marked as "not to be indexed by
+        the content indexing service", if ``notindexed`` is specified
+        for "write" mode access.
+        '''
+        return self.__call__
+
+    def read(self, path):
+        with self(path, 'rb') as fp:
+            return fp.read()
+
+    def readlines(self, path, mode='rb'):
+        with self(path, mode=mode) as fp:
+            return fp.readlines()
+
+    def write(self, path, data, backgroundclose=False):
+        with self(path, 'wb', backgroundclose=backgroundclose) as fp:
+            return fp.write(data)
+
+    def writelines(self, path, data, mode='wb', notindexed=False):
+        with self(path, mode=mode, notindexed=notindexed) as fp:
+            return fp.writelines(data)
+
+    def append(self, path, data):
+        with self(path, 'ab') as fp:
+            return fp.write(data)
+
+    def basename(self, path):
+        """return base element of a path (as os.path.basename would do)
+
+        This exists to allow handling of strange encoding if needed."""
+        return os.path.basename(path)
+
+    def chmod(self, path, mode):
+        return os.chmod(self.join(path), mode)
+
+    def dirname(self, path):
+        """return dirname element of a path (as os.path.dirname would do)
+
+        This exists to allow handling of strange encoding if needed."""
+        return os.path.dirname(path)
+
+    def exists(self, path=None):
+        return os.path.exists(self.join(path))
+
+    def fstat(self, fp):
+        return util.fstat(fp)
+
+    def isdir(self, path=None):
+        return os.path.isdir(self.join(path))
+
+    def isfile(self, path=None):
+        return os.path.isfile(self.join(path))
+
+    def islink(self, path=None):
+        return os.path.islink(self.join(path))
+
+    def isfileorlink(self, path=None):
+        '''return whether path is a regular file or a symlink
+
+        Unlike isfile, this doesn't follow symlinks.'''
+        try:
+            st = self.lstat(path)
+        except OSError:
+            return False
+        mode = st.st_mode
+        return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
+
+    def reljoin(self, *paths):
+        """join various elements of a path together (as os.path.join would do)
+
+        The vfs base is not injected so that path stay relative. This exists
+        to allow handling of strange encoding if needed."""
+        return os.path.join(*paths)
+
+    def split(self, path):
+        """split top-most element of a path (as os.path.split would do)
+
+        This exists to allow handling of strange encoding if needed."""
+        return os.path.split(path)
+
+    def lexists(self, path=None):
+        return os.path.lexists(self.join(path))
+
+    def lstat(self, path=None):
+        return os.lstat(self.join(path))
+
+    def listdir(self, path=None):
+        return os.listdir(self.join(path))
+
+    def makedir(self, path=None, notindexed=True):
+        return util.makedir(self.join(path), notindexed)
+
+    def makedirs(self, path=None, mode=None):
+        return util.makedirs(self.join(path), mode)
+
+    def makelock(self, info, path):
+        return util.makelock(info, self.join(path))
+
+    def mkdir(self, path=None):
+        return os.mkdir(self.join(path))
+
+    def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
+        fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
+                                    dir=self.join(dir), text=text)
+        dname, fname = util.split(name)
+        if dir:
+            return fd, os.path.join(dir, fname)
+        else:
+            return fd, fname
+
+    def readdir(self, path=None, stat=None, skip=None):
+        return osutil.listdir(self.join(path), stat, skip)
+
+    def readlock(self, path):
+        return util.readlock(self.join(path))
+
+    def rename(self, src, dst, checkambig=False):
+        """Rename from src to dst
+
+        checkambig argument is used with util.filestat, and is useful
+        only if destination file is guarded by any lock
+        (e.g. repo.lock or repo.wlock).
+        """
+        dstpath = self.join(dst)
+        oldstat = checkambig and util.filestat(dstpath)
+        if oldstat and oldstat.stat:
+            ret = util.rename(self.join(src), dstpath)
+            newstat = util.filestat(dstpath)
+            if newstat.isambig(oldstat):
+                # stat of renamed file is ambiguous to original one
+                newstat.avoidambig(dstpath, oldstat)
+            return ret
+        return util.rename(self.join(src), dstpath)
+
+    def readlink(self, path):
+        return os.readlink(self.join(path))
+
+    def removedirs(self, path=None):
+        """Remove a leaf directory and all empty intermediate ones
+        """
+        return util.removedirs(self.join(path))
+
+    def rmtree(self, path=None, ignore_errors=False, forcibly=False):
+        """Remove a directory tree recursively
+
+        If ``forcibly``, this tries to remove READ-ONLY files, too.
+        """
+        if forcibly:
+            def onerror(function, path, excinfo):
+                if function is not os.remove:
+                    raise
+                # read-only files cannot be unlinked under Windows
+                s = os.stat(path)
+                if (s.st_mode & stat.S_IWRITE) != 0:
+                    raise
+                os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
+                os.remove(path)
+        else:
+            onerror = None
+        return shutil.rmtree(self.join(path),
+                             ignore_errors=ignore_errors, onerror=onerror)
+
+    def setflags(self, path, l, x):
+        return util.setflags(self.join(path), l, x)
+
+    def stat(self, path=None):
+        return os.stat(self.join(path))
+
+    def unlink(self, path=None):
+        return util.unlink(self.join(path))
+
+    def unlinkpath(self, path=None, ignoremissing=False):
+        return util.unlinkpath(self.join(path), ignoremissing)
+
+    def utime(self, path=None, t=None):
+        return os.utime(self.join(path), t)
+
+    def walk(self, path=None, onerror=None):
+        """Yield (dirpath, dirs, files) tuple for each directories under path
+
+        ``dirpath`` is relative one from the root of this vfs. This
+        uses ``os.sep`` as path separator, even you specify POSIX
+        style ``path``.
+
+        "The root of this vfs" is represented as empty ``dirpath``.
+        """
+        root = os.path.normpath(self.join(None))
+        # when dirpath == root, dirpath[prefixlen:] becomes empty
+        # because len(dirpath) < prefixlen.
+        prefixlen = len(pathutil.normasprefix(root))
+        for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
+            yield (dirpath[prefixlen:], dirs, files)
+
+    @contextlib.contextmanager
+    def backgroundclosing(self, ui, expectedcount=-1):
+        """Allow files to be closed asynchronously.
+
+        When this context manager is active, ``backgroundclose`` can be passed
+        to ``__call__``/``open`` to result in the file possibly being closed
+        asynchronously, on a background thread.
+        """
+        # This is an arbitrary restriction and could be changed if we ever
+        # have a use case.
+        vfs = getattr(self, 'vfs', self)
+        if getattr(vfs, '_backgroundfilecloser', None):
+            raise error.Abort(
+                _('can only have 1 active background file closer'))
+
+        with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
+            try:
+                vfs._backgroundfilecloser = bfc
+                yield bfc
+            finally:
+                vfs._backgroundfilecloser = None
+
+class vfs(abstractvfs):
+    '''Operate files relative to a base directory
+
+    This class is used to hide the details of COW semantics and
+    remote file access from higher level code.
+    '''
+    def __init__(self, base, audit=True, expandpath=False, realpath=False):
+        if expandpath:
+            base = util.expandpath(base)
+        if realpath:
+            base = os.path.realpath(base)
+        self.base = base
+        self.mustaudit = audit
+        self.createmode = None
+        self._trustnlink = None
+
+    @property
+    def mustaudit(self):
+        return self._audit
+
+    @mustaudit.setter
+    def mustaudit(self, onoff):
+        self._audit = onoff
+        if onoff:
+            self.audit = pathutil.pathauditor(self.base)
+        else:
+            self.audit = util.always
+
+    @util.propertycache
+    def _cansymlink(self):
+        return util.checklink(self.base)
+
+    @util.propertycache
+    def _chmod(self):
+        return util.checkexec(self.base)
+
+    def _fixfilemode(self, name):
+        if self.createmode is None or not self._chmod:
+            return
+        os.chmod(name, self.createmode & 0o666)
+
+    def __call__(self, path, mode="r", text=False, atomictemp=False,
+                 notindexed=False, backgroundclose=False, checkambig=False):
+        '''Open ``path`` file, which is relative to vfs root.
+
+        Newly created directories are marked as "not to be indexed by
+        the content indexing service", if ``notindexed`` is specified
+        for "write" mode access.
+
+        If ``backgroundclose`` is passed, the file may be closed asynchronously.
+        It can only be used if the ``self.backgroundclosing()`` context manager
+        is active. This should only be specified if the following criteria hold:
+
+        1. There is a potential for writing thousands of files. Unless you
+           are writing thousands of files, the performance benefits of
+           asynchronously closing files is not realized.
+        2. Files are opened exactly once for the ``backgroundclosing``
+           active duration and are therefore free of race conditions between
+           closing a file on a background thread and reopening it. (If the
+           file were opened multiple times, there could be unflushed data
+           because the original file handle hasn't been flushed/closed yet.)
+
+        ``checkambig`` argument is passed to atomictemplfile (valid
+        only for writing), and is useful only if target file is
+        guarded by any lock (e.g. repo.lock or repo.wlock).
+        '''
+        if self._audit:
+            r = util.checkosfilename(path)
+            if r:
+                raise error.Abort("%s: %r" % (r, path))
+        self.audit(path)
+        f = self.join(path)
+
+        if not text and "b" not in mode:
+            mode += "b" # for that other OS
+
+        nlink = -1
+        if mode not in ('r', 'rb'):
+            dirname, basename = util.split(f)
+            # If basename is empty, then the path is malformed because it points
+            # to a directory. Let the posixfile() call below raise IOError.
+            if basename:
+                if atomictemp:
+                    util.makedirs(dirname, self.createmode, notindexed)
+                    return util.atomictempfile(f, mode, self.createmode,
+                                               checkambig=checkambig)
+                try:
+                    if 'w' in mode:
+                        util.unlink(f)
+                        nlink = 0
+                    else:
+                        # nlinks() may behave differently for files on Windows
+                        # shares if the file is open.
+                        with util.posixfile(f):
+                            nlink = util.nlinks(f)
+                            if nlink < 1:
+                                nlink = 2 # force mktempcopy (issue1922)
+                except (OSError, IOError) as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                    nlink = 0
+                    util.makedirs(dirname, self.createmode, notindexed)
+                if nlink > 0:
+                    if self._trustnlink is None:
+                        self._trustnlink = nlink > 1 or util.checknlink(f)
+                    if nlink > 1 or not self._trustnlink:
+                        util.rename(util.mktempcopy(f), f)
+        fp = util.posixfile(f, mode)
+        if nlink == 0:
+            self._fixfilemode(f)
+
+        if checkambig:
+            if mode in ('r', 'rb'):
+                raise error.Abort(_('implementation error: mode %s is not'
+                                    ' valid for checkambig=True') % mode)
+            fp = checkambigatclosing(fp)
+
+        if backgroundclose:
+            if not self._backgroundfilecloser:
+                raise error.Abort(_('backgroundclose can only be used when a '
+                                  'backgroundclosing context manager is active')
+                                  )
+
+            fp = delayclosedfile(fp, self._backgroundfilecloser)
+
+        return fp
+
+    def symlink(self, src, dst):
+        self.audit(dst)
+        linkname = self.join(dst)
+        try:
+            os.unlink(linkname)
+        except OSError:
+            pass
+
+        util.makedirs(os.path.dirname(linkname), self.createmode)
+
+        if self._cansymlink:
+            try:
+                os.symlink(src, linkname)
+            except OSError as err:
+                raise OSError(err.errno, _('could not symlink to %r: %s') %
+                              (src, err.strerror), linkname)
+        else:
+            self.write(dst, src)
+
+    def join(self, path, *insidef):
+        if path:
+            return os.path.join(self.base, path, *insidef)
+        else:
+            return self.base
+
+opener = vfs
+
+class auditvfs(object):
+    def __init__(self, vfs):
+        self.vfs = vfs
+
+    @property
+    def mustaudit(self):
+        return self.vfs.mustaudit
+
+    @mustaudit.setter
+    def mustaudit(self, onoff):
+        self.vfs.mustaudit = onoff
+
+    @property
+    def options(self):
+        return self.vfs.options
+
+    @options.setter
+    def options(self, value):
+        self.vfs.options = value
+
+class filtervfs(abstractvfs, auditvfs):
+    '''Wrapper vfs for filtering filenames with a function.'''
+
+    def __init__(self, vfs, filter):
+        auditvfs.__init__(self, vfs)
+        self._filter = filter
+
+    def __call__(self, path, *args, **kwargs):
+        return self.vfs(self._filter(path), *args, **kwargs)
+
+    def join(self, path, *insidef):
+        if path:
+            return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
+        else:
+            return self.vfs.join(path)
+
+filteropener = filtervfs
+
+class readonlyvfs(abstractvfs, auditvfs):
+    '''Wrapper vfs preventing any writing.'''
+
+    def __init__(self, vfs):
+        auditvfs.__init__(self, vfs)
+
+    def __call__(self, path, mode='r', *args, **kw):
+        if mode not in ('r', 'rb'):
+            raise error.Abort(_('this vfs is read only'))
+        return self.vfs(path, mode, *args, **kw)
+
+    def join(self, path, *insidef):
+        return self.vfs.join(path, *insidef)
+
+class closewrapbase(object):
+    """Base class of wrapper, which hooks closing
+
+    Do not instantiate outside of the vfs layer.
+    """
+    def __init__(self, fh):
+        object.__setattr__(self, '_origfh', fh)
+
+    def __getattr__(self, attr):
+        return getattr(self._origfh, attr)
+
+    def __setattr__(self, attr, value):
+        return setattr(self._origfh, attr, value)
+
+    def __delattr__(self, attr):
+        return delattr(self._origfh, attr)
+
+    def __enter__(self):
+        return self._origfh.__enter__()
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        raise NotImplementedError('attempted instantiating ' + str(type(self)))
+
+    def close(self):
+        raise NotImplementedError('attempted instantiating ' + str(type(self)))
+
+class delayclosedfile(closewrapbase):
+    """Proxy for a file object whose close is delayed.
+
+    Do not instantiate outside of the vfs layer.
+    """
+    def __init__(self, fh, closer):
+        super(delayclosedfile, self).__init__(fh)
+        object.__setattr__(self, '_closer', closer)
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self._closer.close(self._origfh)
+
+    def close(self):
+        self._closer.close(self._origfh)
+
+class backgroundfilecloser(object):
+    """Coordinates background closing of file handles on multiple threads."""
+    def __init__(self, ui, expectedcount=-1):
+        self._running = False
+        self._entered = False
+        self._threads = []
+        self._threadexception = None
+
+        # Only Windows/NTFS has slow file closing. So only enable by default
+        # on that platform. But allow to be enabled elsewhere for testing.
+        defaultenabled = pycompat.osname == 'nt'
+        enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
+
+        if not enabled:
+            return
+
+        # There is overhead to starting and stopping the background threads.
+        # Don't do background processing unless the file count is large enough
+        # to justify it.
+        minfilecount = ui.configint('worker', 'backgroundcloseminfilecount',
+                                    2048)
+        # FUTURE dynamically start background threads after minfilecount closes.
+        # (We don't currently have any callers that don't know their file count)
+        if expectedcount > 0 and expectedcount < minfilecount:
+            return
+
+        # Windows defaults to a limit of 512 open files. A buffer of 128
+        # should give us enough headway.
+        maxqueue = ui.configint('worker', 'backgroundclosemaxqueue', 384)
+        threadcount = ui.configint('worker', 'backgroundclosethreadcount', 4)
+
+        ui.debug('starting %d threads for background file closing\n' %
+                 threadcount)
+
+        self._queue = util.queue(maxsize=maxqueue)
+        self._running = True
+
+        for i in range(threadcount):
+            t = threading.Thread(target=self._worker, name='backgroundcloser')
+            self._threads.append(t)
+            t.start()
+
+    def __enter__(self):
+        self._entered = True
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self._running = False
+
+        # Wait for threads to finish closing so open files don't linger for
+        # longer than lifetime of context manager.
+        for t in self._threads:
+            t.join()
+
+    def _worker(self):
+        """Main routine for worker thread."""
+        while True:
+            try:
+                fh = self._queue.get(block=True, timeout=0.100)
+                # Need to catch or the thread will terminate and
+                # we could orphan file descriptors.
+                try:
+                    fh.close()
+                except Exception as e:
+                    # Stash so can re-raise from main thread later.
+                    self._threadexception = e
+            except util.empty:
+                if not self._running:
+                    break
+
+    def close(self, fh):
+        """Schedule a file for closing."""
+        if not self._entered:
+            raise error.Abort(_('can only call close() when context manager '
+                              'active'))
+
+        # If a background thread encountered an exception, raise now so we fail
+        # fast. Otherwise we may potentially go on for minutes until the error
+        # is acted on.
+        if self._threadexception:
+            e = self._threadexception
+            self._threadexception = None
+            raise e
+
+        # If we're not actively running, close synchronously.
+        if not self._running:
+            fh.close()
+            return
+
+        self._queue.put(fh, block=True, timeout=None)
+
+class checkambigatclosing(closewrapbase):
+    """Proxy for a file object, to avoid ambiguity of file stat
+
+    See also util.filestat for detail about "ambiguity of file stat".
+
+    This proxy is useful only if the target file is guarded by any
+    lock (e.g. repo.lock or repo.wlock)
+
+    Do not instantiate outside of the vfs layer.
+    """
+    def __init__(self, fh):
+        super(checkambigatclosing, self).__init__(fh)
+        object.__setattr__(self, '_oldstat', util.filestat(fh.name))
+
+    def _checkambig(self):
+        oldstat = self._oldstat
+        if oldstat.stat:
+            newstat = util.filestat(self._origfh.name)
+            if newstat.isambig(oldstat):
+                # stat of changed file is ambiguous to original one
+                newstat.avoidambig(self._origfh.name, oldstat)
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self._origfh.__exit__(exc_type, exc_value, exc_tb)
+        self._checkambig()
+
+    def close(self):
+        self._origfh.close()
+        self._checkambig()