Mercurial > hg

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext/sparse.py	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,1081 @@
+# sparse.py - allow sparse checkouts of the working directory
+#
+# Copyright 2014 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""allow sparse checkouts of the working directory (EXPERIMENTAL)
+"""
+
+from __future__ import absolute_import
+
+import collections
+import hashlib
+import os
+
+from mercurial.i18n import _
+from mercurial.node import nullid
+from mercurial import (
+    cmdutil,
+    commands,
+    context,
+    dirstate,
+    error,
+    extensions,
+    hg,
+    localrepo,
+    match as matchmod,
+    merge as mergemod,
+    registrar,
+    util,
+)
+
+# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = 'ships-with-hg-core'
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+def uisetup(ui):
+    _setupupdates(ui)
+    _setupcommit(ui)
+
+def extsetup(ui):
+    _setupclone(ui)
+    _setuplog(ui)
+    _setupadd(ui)
+    _setupdirstate(ui)
+    # if fsmonitor is enabled, tell it to use our hash function
+    try:
+        fsmonitor = extensions.find('fsmonitor')
+        def _hashignore(orig, ignore):
+            return _hashmatcher(ignore)
+        extensions.wrapfunction(fsmonitor, '_hashignore', _hashignore)
+    except KeyError:
+        pass
+    # do the same for hgwatchman, old name
+    try:
+        hgwatchman = extensions.find('hgwatchman')
+        def _hashignore(orig, ignore):
+            return _hashmatcher(ignore)
+        extensions.wrapfunction(hgwatchman, '_hashignore', _hashignore)
+    except KeyError:
+        pass
+
+def reposetup(ui, repo):
+    if not util.safehasattr(repo, 'dirstate'):
+        return
+
+    _wraprepo(ui, repo)
+
+def replacefilecache(cls, propname, replacement):
+    """Replace a filecache property with a new class. This allows changing the
+    cache invalidation condition."""
+    origcls = cls
+    assert callable(replacement)
+    while cls is not object:
+        if propname in cls.__dict__:
+            orig = cls.__dict__[propname]
+            setattr(cls, propname, replacement(orig))
+            break
+        cls = cls.__bases__[0]
+
+    if cls is object:
+        raise AttributeError(_("type '%s' has no property '%s'") % (origcls,
+                             propname))
+
+def _setupupdates(ui):
+    def _calculateupdates(orig, repo, wctx, mctx, ancestors, branchmerge, *arg,
+                          **kwargs):
+        """Filter updates to only lay out files that match the sparse rules.
+        """
+        actions, diverge, renamedelete = orig(repo, wctx, mctx, ancestors,
+                                              branchmerge, *arg, **kwargs)
+
+        if not util.safehasattr(repo, 'sparsematch'):
+            return actions, diverge, renamedelete
+
+        files = set()
+        prunedactions = {}
+        oldrevs = [pctx.rev() for pctx in wctx.parents()]
+        oldsparsematch = repo.sparsematch(*oldrevs)
+
+        if branchmerge:
+            # If we're merging, use the wctx filter, since we're merging into
+            # the wctx.
+            sparsematch = repo.sparsematch(wctx.parents()[0].rev())
+        else:
+            # If we're updating, use the target context's filter, since we're
+            # moving to the target context.
+            sparsematch = repo.sparsematch(mctx.rev())
+
+        temporaryfiles = []
+        for file, action in actions.iteritems():
+            type, args, msg = action
+            files.add(file)
+            if sparsematch(file):
+                prunedactions[file] = action
+            elif type == 'm':
+                temporaryfiles.append(file)
+                prunedactions[file] = action
+            elif branchmerge:
+                if type != 'k':
+                    temporaryfiles.append(file)
+                    prunedactions[file] = action
+            elif type == 'f':
+                prunedactions[file] = action
+            elif file in wctx:
+                prunedactions[file] = ('r', args, msg)
+
+        if len(temporaryfiles) > 0:
+            ui.status(_("temporarily included %d file(s) in the sparse checkout"
+                " for merging\n") % len(temporaryfiles))
+            repo.addtemporaryincludes(temporaryfiles)
+
+            # Add the new files to the working copy so they can be merged, etc
+            actions = []
+            message = 'temporarily adding to sparse checkout'
+            wctxmanifest = repo[None].manifest()
+            for file in temporaryfiles:
+                if file in wctxmanifest:
+                    fctx = repo[None][file]
+                    actions.append((file, (fctx.flags(), False), message))
+
+            typeactions = collections.defaultdict(list)
+            typeactions['g'] = actions
+            mergemod.applyupdates(repo, typeactions, repo[None], repo['.'],
+                                  False)
+
+            dirstate = repo.dirstate
+            for file, flags, msg in actions:
+                dirstate.normal(file)
+
+        profiles = repo.getactiveprofiles()
+        changedprofiles = profiles & files
+        # If an active profile changed during the update, refresh the checkout.
+        # Don't do this during a branch merge, since all incoming changes should
+        # have been handled by the temporary includes above.
+        if changedprofiles and not branchmerge:
+            mf = mctx.manifest()
+            for file in mf:
+                old = oldsparsematch(file)
+                new = sparsematch(file)
+                if not old and new:
+                    flags = mf.flags(file)
+                    prunedactions[file] = ('g', (flags, False), '')
+                elif old and not new:
+                    prunedactions[file] = ('r', [], '')
+
+        return prunedactions, diverge, renamedelete
+
+    extensions.wrapfunction(mergemod, 'calculateupdates', _calculateupdates)
+
+    def _update(orig, repo, node, branchmerge, *args, **kwargs):
+        results = orig(repo, node, branchmerge, *args, **kwargs)
+
+        # If we're updating to a location, clean up any stale temporary includes
+        # (ex: this happens during hg rebase --abort).
+        if not branchmerge and util.safehasattr(repo, 'sparsematch'):
+            repo.prunetemporaryincludes()
+        return results
+
+    extensions.wrapfunction(mergemod, 'update', _update)
+
+def _setupcommit(ui):
+    def _refreshoncommit(orig, self, node):
+        """Refresh the checkout when commits touch .hgsparse
+        """
+        orig(self, node)
+        repo = self._repo
+        if util.safehasattr(repo, 'getsparsepatterns'):
+            ctx = repo[node]
+            _, _, profiles = repo.getsparsepatterns(ctx.rev())
+            if set(profiles) & set(ctx.files()):
+                origstatus = repo.status()
+                origsparsematch = repo.sparsematch()
+                _refresh(repo.ui, repo, origstatus, origsparsematch, True)
+
+            repo.prunetemporaryincludes()
+
+    extensions.wrapfunction(context.committablectx, 'markcommitted',
+        _refreshoncommit)
+
+def _setuplog(ui):
+    entry = commands.table['^log|history']
+    entry[1].append(('', 'sparse', None,
+        "limit to changesets affecting the sparse checkout"))
+
+    def _logrevs(orig, repo, opts):
+        revs = orig(repo, opts)
+        if opts.get('sparse'):
+            sparsematch = repo.sparsematch()
+            def ctxmatch(rev):
+                ctx = repo[rev]
+                return any(f for f in ctx.files() if sparsematch(f))
+            revs = revs.filter(ctxmatch)
+        return revs
+    extensions.wrapfunction(cmdutil, '_logrevs', _logrevs)
+
+def _clonesparsecmd(orig, ui, repo, *args, **opts):
+    include_pat = opts.get('include')
+    exclude_pat = opts.get('exclude')
+    enableprofile_pat = opts.get('enable_profile')
+    include = exclude = enableprofile = False
+    if include_pat:
+        pat = include_pat
+        include = True
+    if exclude_pat:
+        pat = exclude_pat
+        exclude = True
+    if enableprofile_pat:
+        pat = enableprofile_pat
+        enableprofile = True
+    if sum([include, exclude, enableprofile]) > 1:
+        raise error.Abort(_("too many flags specified."))
+    if include or exclude or enableprofile:
+        def clonesparse(orig, self, node, overwrite, *args, **kwargs):
+            _config(self.ui, self.unfiltered(), pat, {}, include=include,
+                    exclude=exclude, enableprofile=enableprofile)
+            return orig(self, node, overwrite, *args, **kwargs)
+        extensions.wrapfunction(hg, 'updaterepo', clonesparse)
+    return orig(ui, repo, *args, **opts)
+
+def _setupclone(ui):
+    entry = commands.table['^clone']
+    entry[1].append(('', 'enable-profile', [],
+                    'enable a sparse profile'))
+    entry[1].append(('', 'include', [],
+                    'include sparse pattern'))
+    entry[1].append(('', 'exclude', [],
+                    'exclude sparse pattern'))
+    extensions.wrapcommand(commands.table, 'clone', _clonesparsecmd)
+
+def _setupadd(ui):
+    entry = commands.table['^add']
+    entry[1].append(('s', 'sparse', None,
+                    'also include directories of added files in sparse config'))
+
+    def _add(orig, ui, repo, *pats, **opts):
+        if opts.get('sparse'):
+            dirs = set()
+            for pat in pats:
+                dirname, basename = util.split(pat)
+                dirs.add(dirname)
+            _config(ui, repo, list(dirs), opts, include=True)
+        return orig(ui, repo, *pats, **opts)
+
+    extensions.wrapcommand(commands.table, 'add', _add)
+
+def _setupdirstate(ui):
+    """Modify the dirstate to prevent stat'ing excluded files,
+    and to prevent modifications to files outside the checkout.
+    """
+
+    def _dirstate(orig, repo):
+        dirstate = orig(repo)
+        dirstate.repo = repo
+        return dirstate
+    extensions.wrapfunction(
+        localrepo.localrepository.dirstate, 'func', _dirstate)
+
+    # The atrocity below is needed to wrap dirstate._ignore. It is a cached
+    # property, which means normal function wrapping doesn't work.
+    class ignorewrapper(object):
+        def __init__(self, orig):
+            self.orig = orig
+            self.origignore = None
+            self.func = None
+            self.sparsematch = None
+
+        def __get__(self, obj, type=None):
+            repo = obj.repo
+            origignore = self.orig.__get__(obj)
+            if not util.safehasattr(repo, 'sparsematch'):
+                return origignore
+
+            sparsematch = repo.sparsematch()
+            if self.sparsematch != sparsematch or self.origignore != origignore:
+                self.func = unionmatcher([origignore,
+                                          negatematcher(sparsematch)])
+                self.sparsematch = sparsematch
+                self.origignore = origignore
+            return self.func
+
+        def __set__(self, obj, value):
+            return self.orig.__set__(obj, value)
+
+        def __delete__(self, obj):
+            return self.orig.__delete__(obj)
+
+    replacefilecache(dirstate.dirstate, '_ignore', ignorewrapper)
+
+    # dirstate.rebuild should not add non-matching files
+    def _rebuild(orig, self, parent, allfiles, changedfiles=None):
+        if util.safehasattr(self.repo, 'sparsematch'):
+            matcher = self.repo.sparsematch()
+            allfiles = allfiles.matches(matcher)
+            if changedfiles:
+                changedfiles = [f for f in changedfiles if matcher(f)]
+
+            if changedfiles is not None:
+                # In _rebuild, these files will be deleted from the dirstate
+                # when they are not found to be in allfiles
+                dirstatefilestoremove = set(f for f in self if not matcher(f))
+                changedfiles = dirstatefilestoremove.union(changedfiles)
+
+        return orig(self, parent, allfiles, changedfiles)
+    extensions.wrapfunction(dirstate.dirstate, 'rebuild', _rebuild)
+
+    # Prevent adding files that are outside the sparse checkout
+    editfuncs = ['normal', 'add', 'normallookup', 'copy', 'remove', 'merge']
+    hint = _('include file with `hg sparse --include <pattern>` or use ' +
+             '`hg add -s <file>` to include file directory while adding')
+    for func in editfuncs:
+        def _wrapper(orig, self, *args):
+            repo = self.repo
+            if util.safehasattr(repo, 'sparsematch'):
+                dirstate = repo.dirstate
+                sparsematch = repo.sparsematch()
+                for f in args:
+                    if (f is not None and not sparsematch(f) and
+                        f not in dirstate):
+                        raise error.Abort(_("cannot add '%s' - it is outside "
+                                            "the sparse checkout") % f,
+                                          hint=hint)
+            return orig(self, *args)
+        extensions.wrapfunction(dirstate.dirstate, func, _wrapper)
+
+def _wraprepo(ui, repo):
+    class SparseRepo(repo.__class__):
+        def readsparseconfig(self, raw):
+            """Takes a string sparse config and returns the includes,
+            excludes, and profiles it specified.
+            """
+            includes = set()
+            excludes = set()
+            current = includes
+            profiles = []
+            for line in raw.split('\n'):
+                line = line.strip()
+                if not line or line.startswith('#'):
+                    # empty or comment line, skip
+                    continue
+                elif line.startswith('%include '):
+                    line = line[9:].strip()
+                    if line:
+                        profiles.append(line)
+                elif line == '[include]':
+                    if current != includes:
+                        raise error.Abort(_('.hg/sparse cannot have includes ' +
+                            'after excludes'))
+                    continue
+                elif line == '[exclude]':
+                    current = excludes
+                elif line:
+                    if line.strip().startswith('/'):
+                        self.ui.warn(_('warning: sparse profile cannot use' +
+                                       ' paths starting with /, ignoring %s\n')
+                                       % line)
+                        continue
+                    current.add(line)
+
+            return includes, excludes, profiles
+
+        def getsparsepatterns(self, rev):
+            """Returns the include/exclude patterns specified by the
+            given rev.
+            """
+            if not self.vfs.exists('sparse'):
+                return set(), set(), []
+            if rev is None:
+                raise error.Abort(_("cannot parse sparse patterns from " +
+                    "working copy"))
+
+            raw = self.vfs.read('sparse')
+            includes, excludes, profiles = self.readsparseconfig(raw)
+
+            ctx = self[rev]
+            if profiles:
+                visited = set()
+                while profiles:
+                    profile = profiles.pop()
+                    if profile in visited:
+                        continue
+                    visited.add(profile)
+
+                    try:
+                        raw = self.getrawprofile(profile, rev)
+                    except error.ManifestLookupError:
+                        msg = (
+                            "warning: sparse profile '%s' not found "
+                            "in rev %s - ignoring it\n" % (profile, ctx))
+                        if self.ui.configbool(
+                                'sparse', 'missingwarning', True):
+                            self.ui.warn(msg)
+                        else:
+                            self.ui.debug(msg)
+                        continue
+                    pincludes, pexcludes, subprofs = \
+                        self.readsparseconfig(raw)
+                    includes.update(pincludes)
+                    excludes.update(pexcludes)
+                    for subprofile in subprofs:
+                        profiles.append(subprofile)
+
+                profiles = visited
+
+            if includes:
+                includes.add('.hg*')
+            return includes, excludes, profiles
+
+        def getrawprofile(self, profile, changeid):
+            try:
+                simplecache = extensions.find('simplecache')
+                node = self[changeid].hex()
+                def func():
+                    return self.filectx(profile, changeid=changeid).data()
+                key = 'sparseprofile:%s:%s' % (profile.replace('/', '__'), node)
+                return simplecache.memoize(func, key,
+                        simplecache.stringserializer, self.ui)
+            except KeyError:
+                return self.filectx(profile, changeid=changeid).data()
+
+        def sparsechecksum(self, filepath):
+            fh = open(filepath)
+            return hashlib.sha1(fh.read()).hexdigest()
+
+        def _sparsesignature(self, includetemp=True):
+            """Returns the signature string representing the contents of the
+            current project sparse configuration. This can be used to cache the
+            sparse matcher for a given set of revs."""
+            signaturecache = self.signaturecache
+            signature = signaturecache.get('signature')
+            if includetemp:
+                tempsignature = signaturecache.get('tempsignature')
+            else:
+                tempsignature = 0
+
+            if signature is None or (includetemp and tempsignature is None):
+                signature = 0
+                try:
+                    sparsepath = self.vfs.join('sparse')
+                    signature = self.sparsechecksum(sparsepath)
+                except (OSError, IOError):
+                    pass
+                signaturecache['signature'] = signature
+
+                tempsignature = 0
+                if includetemp:
+                    try:
+                        tempsparsepath = self.vfs.join('tempsparse')
+                        tempsignature = self.sparsechecksum(tempsparsepath)
+                    except (OSError, IOError):
+                        pass
+                    signaturecache['tempsignature'] = tempsignature
+            return '%s %s' % (str(signature), str(tempsignature))
+
+        def invalidatecaches(self):
+            self.invalidatesignaturecache()
+            return super(SparseRepo, self).invalidatecaches()
+
+        def invalidatesignaturecache(self):
+            self.signaturecache.clear()
+
+        def sparsematch(self, *revs, **kwargs):
+            """Returns the sparse match function for the given revs.
+
+            If multiple revs are specified, the match function is the union
+            of all the revs.
+
+            `includetemp` is used to indicate if the temporarily included file
+            should be part of the matcher.
+            """
+            if not revs or revs == (None,):
+                revs = [self.changelog.rev(node) for node in
+                    self.dirstate.parents() if node != nullid]
+
+            includetemp = kwargs.get('includetemp', True)
+            signature = self._sparsesignature(includetemp=includetemp)
+
+            key = '%s %s' % (str(signature), ' '.join([str(r) for r in revs]))
+
+            result = self.sparsecache.get(key, None)
+            if result:
+                return result
+
+            matchers = []
+            for rev in revs:
+                try:
+                    includes, excludes, profiles = self.getsparsepatterns(rev)
+
+                    if includes or excludes:
+                        # Explicitly include subdirectories of includes so
+                        # status will walk them down to the actual include.
+                        subdirs = set()
+                        for include in includes:
+                            dirname = os.path.dirname(include)
+                            # basename is used to avoid issues with absolute
+                            # paths (which on Windows can include the drive).
+                            while os.path.basename(dirname):
+                                subdirs.add(dirname)
+                                dirname = os.path.dirname(dirname)
+
+                        matcher = matchmod.match(self.root, '', [],
+                            include=includes, exclude=excludes,
+                            default='relpath')
+                        if subdirs:
+                            matcher = forceincludematcher(matcher, subdirs)
+                        matchers.append(matcher)
+                except IOError:
+                    pass
+
+            result = None
+            if not matchers:
+                result = matchmod.always(self.root, '')
+            elif len(matchers) == 1:
+                result = matchers[0]
+            else:
+                result = unionmatcher(matchers)
+
+            if kwargs.get('includetemp', True):
+                tempincludes = self.gettemporaryincludes()
+                result = forceincludematcher(result, tempincludes)
+
+            self.sparsecache[key] = result
+
+            return result
+
+        def getactiveprofiles(self):
+            revs = [self.changelog.rev(node) for node in
+                    self.dirstate.parents() if node != nullid]
+
+            activeprofiles = set()
+            for rev in revs:
+                _, _, profiles = self.getsparsepatterns(rev)
+                activeprofiles.update(profiles)
+
+            return activeprofiles
+
+        def writesparseconfig(self, include, exclude, profiles):
+            raw = '%s[include]\n%s\n[exclude]\n%s\n' % (
+                ''.join(['%%include %s\n' % p for p in sorted(profiles)]),
+                '\n'.join(sorted(include)),
+                '\n'.join(sorted(exclude)))
+            self.vfs.write("sparse", raw)
+            self.invalidatesignaturecache()
+
+        def addtemporaryincludes(self, files):
+            includes = self.gettemporaryincludes()
+            for file in files:
+                includes.add(file)
+            self._writetemporaryincludes(includes)
+
+        def gettemporaryincludes(self):
+            existingtemp = set()
+            if self.vfs.exists('tempsparse'):
+                raw = self.vfs.read('tempsparse')
+                existingtemp.update(raw.split('\n'))
+            return existingtemp
+
+        def _writetemporaryincludes(self, includes):
+            raw = '\n'.join(sorted(includes))
+            self.vfs.write('tempsparse', raw)
+            self.invalidatesignaturecache()
+
+        def prunetemporaryincludes(self):
+            if repo.vfs.exists('tempsparse'):
+                origstatus = self.status()
+                modified, added, removed, deleted, a, b, c = origstatus
+                if modified or added or removed or deleted:
+                    # Still have pending changes. Don't bother trying to prune.
+                    return
+
+                sparsematch = self.sparsematch(includetemp=False)
+                dirstate = self.dirstate
+                actions = []
+                dropped = []
+                tempincludes = self.gettemporaryincludes()
+                for file in tempincludes:
+                    if file in dirstate and not sparsematch(file):
+                        message = 'dropping temporarily included sparse files'
+                        actions.append((file, None, message))
+                        dropped.append(file)
+
+                typeactions = collections.defaultdict(list)
+                typeactions['r'] = actions
+                mergemod.applyupdates(self, typeactions, self[None], self['.'],
+                                      False)
+
+                # Fix dirstate
+                for file in dropped:
+                    dirstate.drop(file)
+
+                self.vfs.unlink('tempsparse')
+                self.invalidatesignaturecache()
+                msg = _("cleaned up %d temporarily added file(s) from the "
+                        "sparse checkout\n")
+                ui.status(msg % len(tempincludes))
+
+    if 'dirstate' in repo._filecache:
+        repo.dirstate.repo = repo
+    repo.sparsecache = {}
+    repo.signaturecache = {}
+    repo.__class__ = SparseRepo
+
+@command('^sparse', [
+    ('I', 'include', False, _('include files in the sparse checkout')),
+    ('X', 'exclude', False, _('exclude files in the sparse checkout')),
+    ('d', 'delete', False, _('delete an include/exclude rule')),
+    ('f', 'force', False, _('allow changing rules even with pending changes')),
+    ('', 'enable-profile', False, _('enables the specified profile')),
+    ('', 'disable-profile', False, _('disables the specified profile')),
+    ('', 'import-rules', False, _('imports rules from a file')),
+    ('', 'clear-rules', False, _('clears local include/exclude rules')),
+    ('', 'refresh', False, _('updates the working after sparseness changes')),
+    ('', 'reset', False, _('makes the repo full again')),
+    ] + commands.templateopts,
+    _('[--OPTION] PATTERN...'))
+def sparse(ui, repo, *pats, **opts):
+    """make the current checkout sparse, or edit the existing checkout
+
+    The sparse command is used to make the current checkout sparse.
+    This means files that don't meet the sparse condition will not be
+    written to disk, or show up in any working copy operations. It does
+    not affect files in history in any way.
+
+    Passing no arguments prints the currently applied sparse rules.
+
+    --include and --exclude are used to add and remove files from the sparse
+    checkout. The effects of adding an include or exclude rule are applied
+    immediately. If applying the new rule would cause a file with pending
+    changes to be added or removed, the command will fail. Pass --force to
+    force a rule change even with pending changes (the changes on disk will
+    be preserved).
+
+    --delete removes an existing include/exclude rule. The effects are
+    immediate.
+
+    --refresh refreshes the files on disk based on the sparse rules. This is
+    only necessary if .hg/sparse was changed by hand.
+
+    --enable-profile and --disable-profile accept a path to a .hgsparse file.
+    This allows defining sparse checkouts and tracking them inside the
+    repository. This is useful for defining commonly used sparse checkouts for
+    many people to use. As the profile definition changes over time, the sparse
+    checkout will automatically be updated appropriately, depending on which
+    changeset is checked out. Changes to .hgsparse are not applied until they
+    have been committed.
+
+    --import-rules accepts a path to a file containing rules in the .hgsparse
+    format, allowing you to add --include, --exclude and --enable-profile rules
+    in bulk. Like the --include, --exclude and --enable-profile switches, the
+    changes are applied immediately.
+
+    --clear-rules removes all local include and exclude rules, while leaving
+    any enabled profiles in place.
+
+    Returns 0 if editing the sparse checkout succeeds.
+    """
+    include = opts.get('include')
+    exclude = opts.get('exclude')
+    force = opts.get('force')
+    enableprofile = opts.get('enable_profile')
+    disableprofile = opts.get('disable_profile')
+    importrules = opts.get('import_rules')
+    clearrules = opts.get('clear_rules')
+    delete = opts.get('delete')
+    refresh = opts.get('refresh')
+    reset = opts.get('reset')
+    count = sum([include, exclude, enableprofile, disableprofile, delete,
+                 importrules, refresh, clearrules, reset])
+    if count > 1:
+        raise error.Abort(_("too many flags specified"))
+
+    if count == 0:
+        if repo.vfs.exists('sparse'):
+            ui.status(repo.vfs.read("sparse") + "\n")
+            temporaryincludes = repo.gettemporaryincludes()
+            if temporaryincludes:
+                ui.status(_("Temporarily Included Files (for merge/rebase):\n"))
+                ui.status(("\n".join(temporaryincludes) + "\n"))
+        else:
+            ui.status(_('repo is not sparse\n'))
+        return
+
+    if include or exclude or delete or reset or enableprofile or disableprofile:
+        _config(ui, repo, pats, opts, include=include, exclude=exclude,
+                reset=reset, delete=delete, enableprofile=enableprofile,
+                disableprofile=disableprofile, force=force)
+
+    if importrules:
+        _import(ui, repo, pats, opts, force=force)
+
+    if clearrules:
+        _clear(ui, repo, pats, force=force)
+
+    if refresh:
+        try:
+            wlock = repo.wlock()
+            fcounts = map(
+                len,
+                _refresh(ui, repo, repo.status(), repo.sparsematch(), force))
+            _verbose_output(ui, opts, 0, 0, 0, *fcounts)
+        finally:
+            wlock.release()
+
+def _config(ui, repo, pats, opts, include=False, exclude=False, reset=False,
+            delete=False, enableprofile=False, disableprofile=False,
+            force=False):
+    """
+    Perform a sparse config update. Only one of the kwargs may be specified.
+    """
+    wlock = repo.wlock()
+    try:
+        oldsparsematch = repo.sparsematch()
+
+        if repo.vfs.exists('sparse'):
+            raw = repo.vfs.read('sparse')
+            oldinclude, oldexclude, oldprofiles = map(
+                set, repo.readsparseconfig(raw))
+        else:
+            oldinclude = set()
+            oldexclude = set()
+            oldprofiles = set()
+
+        try:
+            if reset:
+                newinclude = set()
+                newexclude = set()
+                newprofiles = set()
+            else:
+                newinclude = set(oldinclude)
+                newexclude = set(oldexclude)
+                newprofiles = set(oldprofiles)
+
+            oldstatus = repo.status()
+
+            if any(pat.startswith('/') for pat in pats):
+                ui.warn(_('warning: paths cannot start with /, ignoring: %s\n')
+                          % ([pat for pat in pats if pat.startswith('/')]))
+            elif include:
+                newinclude.update(pats)
+            elif exclude:
+                newexclude.update(pats)
+            elif enableprofile:
+                newprofiles.update(pats)
+            elif disableprofile:
+                newprofiles.difference_update(pats)
+            elif delete:
+                newinclude.difference_update(pats)
+                newexclude.difference_update(pats)
+
+            repo.writesparseconfig(newinclude, newexclude, newprofiles)
+            fcounts = map(
+                len, _refresh(ui, repo, oldstatus, oldsparsematch, force))
+
+            profilecount = (len(newprofiles - oldprofiles) -
+                            len(oldprofiles - newprofiles))
+            includecount = (len(newinclude - oldinclude) -
+                            len(oldinclude - newinclude))
+            excludecount = (len(newexclude - oldexclude) -
+                            len(oldexclude - newexclude))
+            _verbose_output(
+                ui, opts, profilecount, includecount, excludecount, *fcounts)
+        except Exception:
+            repo.writesparseconfig(oldinclude, oldexclude, oldprofiles)
+            raise
+    finally:
+        wlock.release()
+
+def _import(ui, repo, files, opts, force=False):
+    with repo.wlock():
+        # load union of current active profile
+        revs = [repo.changelog.rev(node) for node in
+                repo.dirstate.parents() if node != nullid]
+
+        # read current configuration
+        raw = ''
+        if repo.vfs.exists('sparse'):
+            raw = repo.vfs.read('sparse')
+        oincludes, oexcludes, oprofiles = repo.readsparseconfig(raw)
+        includes, excludes, profiles = map(
+                set, (oincludes, oexcludes, oprofiles))
+
+        # all active rules
+        aincludes, aexcludes, aprofiles = set(), set(), set()
+        for rev in revs:
+            rincludes, rexcludes, rprofiles = repo.getsparsepatterns(rev)
+            aincludes.update(rincludes)
+            aexcludes.update(rexcludes)
+            aprofiles.update(rprofiles)
+
+        # import rules on top; only take in rules that are not yet
+        # part of the active rules.
+        changed = False
+        for file in files:
+            with util.posixfile(util.expandpath(file)) as importfile:
+                iincludes, iexcludes, iprofiles = repo.readsparseconfig(
+                    importfile.read())
+                oldsize = len(includes) + len(excludes) + len(profiles)
+                includes.update(iincludes - aincludes)
+                excludes.update(iexcludes - aexcludes)
+                profiles.update(set(iprofiles) - aprofiles)
+                if len(includes) + len(excludes) + len(profiles) > oldsize:
+                    changed = True
+
+        profilecount = includecount = excludecount = 0
+        fcounts = (0, 0, 0)
+
+        if changed:
+            profilecount = len(profiles - aprofiles)
+            includecount = len(includes - aincludes)
+            excludecount = len(excludes - aexcludes)
+
+            oldstatus = repo.status()
+            oldsparsematch = repo.sparsematch()
+            repo.writesparseconfig(includes, excludes, profiles)
+
+            try:
+                fcounts = map(
+                    len, _refresh(ui, repo, oldstatus, oldsparsematch, force))
+            except Exception:
+                repo.writesparseconfig(oincludes, oexcludes, oprofiles)
+                raise
+
+        _verbose_output(ui, opts, profilecount, includecount, excludecount,
+                        *fcounts)
+
+def _clear(ui, repo, files, force=False):
+    with repo.wlock():
+        raw = ''
+        if repo.vfs.exists('sparse'):
+            raw = repo.vfs.read('sparse')
+        includes, excludes, profiles = repo.readsparseconfig(raw)
+
+        if includes or excludes:
+            oldstatus = repo.status()
+            oldsparsematch = repo.sparsematch()
+            repo.writesparseconfig(set(), set(), profiles)
+            _refresh(ui, repo, oldstatus, oldsparsematch, force)
+
+def _refresh(ui, repo, origstatus, origsparsematch, force):
+    """Refreshes which files are on disk by comparing the old status and
+    sparsematch with the new sparsematch.
+
+    Will raise an exception if a file with pending changes is being excluded
+    or included (unless force=True).
+    """
+    modified, added, removed, deleted, unknown, ignored, clean = origstatus
+
+    # Verify there are no pending changes
+    pending = set()
+    pending.update(modified)
+    pending.update(added)
+    pending.update(removed)
+    sparsematch = repo.sparsematch()
+    abort = False
+    for file in pending:
+        if not sparsematch(file):
+            ui.warn(_("pending changes to '%s'\n") % file)
+            abort = not force
+    if abort:
+        raise error.Abort(_("could not update sparseness due to " +
+            "pending changes"))
+
+    # Calculate actions
+    dirstate = repo.dirstate
+    ctx = repo['.']
+    added = []
+    lookup = []
+    dropped = []
+    mf = ctx.manifest()
+    files = set(mf)
+
+    actions = {}
+
+    for file in files:
+        old = origsparsematch(file)
+        new = sparsematch(file)
+        # Add files that are newly included, or that don't exist in
+        # the dirstate yet.
+        if (new and not old) or (old and new and not file in dirstate):
+            fl = mf.flags(file)
+            if repo.wvfs.exists(file):
+                actions[file] = ('e', (fl,), '')
+                lookup.append(file)
+            else:
+                actions[file] = ('g', (fl, False), '')
+                added.append(file)
+        # Drop files that are newly excluded, or that still exist in
+        # the dirstate.
+        elif (old and not new) or (not old and not new and file in dirstate):
+            dropped.append(file)
+            if file not in pending:
+                actions[file] = ('r', [], '')
+
+    # Verify there are no pending changes in newly included files
+    abort = False
+    for file in lookup:
+        ui.warn(_("pending changes to '%s'\n") % file)
+        abort = not force
+    if abort:
+        raise error.Abort(_("cannot change sparseness due to " +
+            "pending changes (delete the files or use --force " +
+            "to bring them back dirty)"))
+
+    # Check for files that were only in the dirstate.
+    for file, state in dirstate.iteritems():
+        if not file in files:
+            old = origsparsematch(file)
+            new = sparsematch(file)
+            if old and not new:
+                dropped.append(file)
+
+    # Apply changes to disk
+    typeactions = dict((m, []) for m in 'a f g am cd dc r dm dg m e k'.split())
+    for f, (m, args, msg) in actions.iteritems():
+        if m not in typeactions:
+            typeactions[m] = []
+        typeactions[m].append((f, args, msg))
+    mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False)
+
+    # Fix dirstate
+    for file in added:
+        dirstate.normal(file)
+
+    for file in dropped:
+        dirstate.drop(file)
+
+    for file in lookup:
+        # File exists on disk, and we're bringing it back in an unknown state.
+        dirstate.normallookup(file)
+
+    return added, dropped, lookup
+
+def _verbose_output(ui, opts, profilecount, includecount, excludecount, added,
+                    dropped, lookup):
+    """Produce --verbose and templatable output
+
+    This specifically enables -Tjson, providing machine-readable stats on how
+    the sparse profile changed.
+
+    """
+    with ui.formatter('sparse', opts) as fm:
+        fm.startitem()
+        fm.condwrite(ui.verbose, 'profiles_added', 'Profile # change: %d\n',
+                     profilecount)
+        fm.condwrite(ui.verbose, 'include_rules_added',
+                     'Include rule # change: %d\n', includecount)
+        fm.condwrite(ui.verbose, 'exclude_rules_added',
+                     'Exclude rule # change: %d\n', excludecount)
+        # In 'plain' verbose mode, mergemod.applyupdates already outputs what
+        # files are added or removed outside of the templating formatter
+        # framework. No point in repeating ourselves in that case.
+        if not fm.isplain():
+            fm.condwrite(ui.verbose, 'files_added', 'Files added: %d\n',
+                         added)
+            fm.condwrite(ui.verbose, 'files_dropped', 'Files dropped: %d\n',
+                         dropped)
+            fm.condwrite(ui.verbose, 'files_conflicting',
+                         'Files conflicting: %d\n', lookup)
+
+class forceincludematcher(object):
+    """A matcher that returns true for any of the forced includes before testing
+    against the actual matcher."""
+    def __init__(self, matcher, includes):
+        self._matcher = matcher
+        self._includes = includes
+
+    def __call__(self, value):
+        return value in self._includes or self._matcher(value)
+
+    def always(self):
+        return False
+
+    def files(self):
+        return []
+
+    def isexact(self):
+        return False
+
+    def anypats(self):
+        return True
+
+    def prefix(self):
+        return False
+
+    def hash(self):
+        sha1 = hashlib.sha1()
+        sha1.update(_hashmatcher(self._matcher))
+        for include in sorted(self._includes):
+            sha1.update(include + '\0')
+        return sha1.hexdigest()
+
+class unionmatcher(object):
+    """A matcher that is the union of several matchers."""
+    def __init__(self, matchers):
+        self._matchers = matchers
+
+    def __call__(self, value):
+        for match in self._matchers:
+            if match(value):
+                return True
+        return False
+
+    def always(self):
+        return False
+
+    def files(self):
+        return []
+
+    def isexact(self):
+        return False
+
+    def anypats(self):
+        return True
+
+    def prefix(self):
+        return False
+
+    def hash(self):
+        sha1 = hashlib.sha1()
+        for m in self._matchers:
+            sha1.update(_hashmatcher(m))
+        return sha1.hexdigest()
+
+class negatematcher(object):
+    def __init__(self, matcher):
+        self._matcher = matcher
+
+    def __call__(self, value):
+        return not self._matcher(value)
+
+    def always(self):
+        return False
+
+    def files(self):
+        return []
+
+    def isexact(self):
+        return False
+
+    def anypats(self):
+        return True
+
+    def hash(self):
+        sha1 = hashlib.sha1()
+        sha1.update('negate')
+        sha1.update(_hashmatcher(self._matcher))
+        return sha1.hexdigest()
+
+def _hashmatcher(matcher):
+    if util.safehasattr(matcher, 'hash'):
+        return matcher.hash()
+
+    sha1 = hashlib.sha1()
+    sha1.update(repr(matcher))
+    return sha1.hexdigest()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-clear.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,73 @@
+test sparse
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat >> $HGRCPATH <<EOF
+  > [extensions]
+  > sparse=
+  > purge=
+  > strip=
+  > rebase=
+  > EOF
+
+  $ echo a > index.html
+  $ echo x > data.py
+  $ echo z > readme.txt
+  $ cat > base.sparse <<EOF
+  > [include]
+  > *.sparse
+  > EOF
+  $ hg ci -Aqm 'initial'
+  $ cat > webpage.sparse <<EOF
+  > %include base.sparse
+  > [include]
+  > *.html
+  > EOF
+  $ hg ci -Aqm 'initial'
+
+Clear rules when there are includes
+
+  $ hg sparse --include *.py
+  $ ls
+  data.py
+  $ hg sparse --clear-rules
+  $ ls
+  base.sparse
+  data.py
+  index.html
+  readme.txt
+  webpage.sparse
+
+Clear rules when there are excludes
+
+  $ hg sparse --exclude *.sparse
+  $ ls
+  data.py
+  index.html
+  readme.txt
+  $ hg sparse --clear-rules
+  $ ls
+  base.sparse
+  data.py
+  index.html
+  readme.txt
+  webpage.sparse
+
+Clearing rules should not alter profiles
+
+  $ hg sparse --enable-profile webpage.sparse
+  $ ls
+  base.sparse
+  index.html
+  webpage.sparse
+  $ hg sparse --include *.py
+  $ ls
+  base.sparse
+  data.py
+  index.html
+  webpage.sparse
+  $ hg sparse --clear-rules
+  $ ls
+  base.sparse
+  index.html
+  webpage.sparse
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-clone.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,72 @@
+test sparse
+
+  $ cat >> $HGRCPATH << EOF
+  > [ui]
+  > ssh = python "$RUNTESTDIR/dummyssh"
+  > username = nobody <no.reply@fb.com>
+  > [extensions]
+  > sparse=
+  > purge=
+  > strip=
+  > rebase=
+  > EOF
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ echo a > index.html
+  $ echo x > data.py
+  $ echo z > readme.txt
+  $ cat > webpage.sparse <<EOF
+  > [include]
+  > *.html
+  > EOF
+  $ cat > backend.sparse <<EOF
+  > [include]
+  > *.py
+  > EOF
+  $ hg ci -Aqm 'initial'
+  $ cd ..
+
+Verify local clone with a sparse profile works
+
+  $ hg clone --enable-profile webpage.sparse myrepo clone1
+  updating to branch default
+  warning: sparse profile 'webpage.sparse' not found in rev 000000000000 - ignoring it
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd clone1
+  $ ls
+  index.html
+  $ cd ..
+
+Verify local clone with include works
+
+  $ hg clone --include *.sparse myrepo clone2
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd clone2
+  $ ls
+  backend.sparse
+  webpage.sparse
+  $ cd ..
+
+Verify local clone with exclude works
+
+  $ hg clone --exclude data.py myrepo clone3
+  updating to branch default
+  4 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd clone3
+  $ ls
+  backend.sparse
+  index.html
+  readme.txt
+  webpage.sparse
+  $ cd ..
+
+Verify sparse clone profile over ssh works
+
+  $ hg clone -q --enable-profile webpage.sparse ssh://user@dummy/myrepo clone4
+  warning: sparse profile 'webpage.sparse' not found in rev 000000000000 - ignoring it
+  $ cd clone4
+  $ ls
+  index.html
+  $ cd ..
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-fsmonitor.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,44 @@
+This test doesn't yet work due to the way fsmonitor is integrated with test runner
+
+  $ exit 80
+
+test sparse interaction with other extensions
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > strip=
+  > EOF
+
+Test fsmonitor integration (if available)
+TODO: make fully isolated integration test a'la https://github.com/facebook/watchman/blob/master/tests/integration/WatchmanInstance.py
+(this one is using the systemwide watchman instance)
+
+  $ touch .watchmanconfig
+  $ echo "ignoredir1/" >> .hgignore
+  $ hg commit -Am ignoredir1
+  adding .hgignore
+  $ echo "ignoredir2/" >> .hgignore
+  $ hg commit -m ignoredir2
+
+  $ hg sparse --reset
+  $ hg sparse -I ignoredir1 -I ignoredir2 -I dir1
+
+  $ mkdir ignoredir1 ignoredir2 dir1
+  $ touch ignoredir1/file ignoredir2/file dir1/file
+
+Run status twice to compensate for a condition in fsmonitor where it will check
+ignored files the second time it runs, regardless of previous state (ask @sid0)
+  $ hg status --config extensions.fsmonitor=
+  ? dir1/file
+  $ hg status --config extensions.fsmonitor=
+  ? dir1/file
+
+Test that fsmonitor ignore hash check updates when .hgignore changes
+
+  $ hg up -q ".^"
+  $ hg status --config extensions.fsmonitor=
+  ? dir1/file
+  ? ignoredir2/file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-import.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,186 @@
+test sparse
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat >> $HGRCPATH <<EOF
+  > [extensions]
+  > sparse=
+  > purge=
+  > strip=
+  > rebase=
+  > EOF
+
+  $ echo a > index.html
+  $ echo x > data.py
+  $ echo z > readme.txt
+  $ cat > base.sparse <<EOF
+  > [include]
+  > *.sparse
+  > EOF
+  $ hg ci -Aqm 'initial'
+  $ cat > webpage.sparse <<EOF
+  > %include base.sparse
+  > [include]
+  > *.html
+  > EOF
+  $ hg ci -Aqm 'initial'
+
+Import a rules file against a 'blank' sparse profile
+
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > [include]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import
+  $ ls
+  data.py
+
+  $ hg sparse --reset
+  $ rm .hg/sparse
+
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > %include base.sparse
+  > [include]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import
+  $ ls
+  base.sparse
+  data.py
+  webpage.sparse
+
+  $ hg sparse --reset
+  $ rm .hg/sparse
+
+Start against an existing profile; rules *already active* should be ignored
+
+  $ hg sparse --enable-profile webpage.sparse
+  $ hg sparse --include *.py
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > %include base.sparse
+  > [include]
+  > *.html
+  > *.txt
+  > [exclude]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import
+  $ ls
+  base.sparse
+  index.html
+  readme.txt
+  webpage.sparse
+  $ cat .hg/sparse
+  %include webpage.sparse
+  [include]
+  *.py
+  *.txt
+  [exclude]
+  *.py
+
+  $ hg sparse --reset
+  $ rm .hg/sparse
+
+Same tests, with -Tjson enabled to output summaries
+
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > [include]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import -Tjson
+  [
+   {
+    "exclude_rules_added": 0,
+    "files_added": 0,
+    "files_conflicting": 0,
+    "files_dropped": 4,
+    "include_rules_added": 1,
+    "profiles_added": 0
+   }
+  ]
+
+  $ hg sparse --reset
+  $ rm .hg/sparse
+
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > %include base.sparse
+  > [include]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import -Tjson
+  [
+   {
+    "exclude_rules_added": 0,
+    "files_added": 0,
+    "files_conflicting": 0,
+    "files_dropped": 2,
+    "include_rules_added": 1,
+    "profiles_added": 1
+   }
+  ]
+
+  $ hg sparse --reset
+  $ rm .hg/sparse
+
+  $ hg sparse --enable-profile webpage.sparse
+  $ hg sparse --include *.py
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > %include base.sparse
+  > [include]
+  > *.html
+  > *.txt
+  > [exclude]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import -Tjson
+  [
+   {
+    "exclude_rules_added": 1,
+    "files_added": 1,
+    "files_conflicting": 0,
+    "files_dropped": 1,
+    "include_rules_added": 1,
+    "profiles_added": 0
+   }
+  ]
+
+If importing results in no new rules being added, no refresh should take place!
+
+  $ cat > $TESTTMP/trap_sparse_refresh.py <<EOF
+  > from mercurial import error, extensions
+  > def extsetup(ui):
+  >     def abort_refresh(ui, *args):
+  >         raise error.Abort('sparse._refresh called!')
+  >     def sparseloaded(loaded):
+  >         if not loaded:
+  >             return
+  >         sparse = extensions.find('sparse')
+  >         sparse._refresh = abort_refresh
+  >     extensions.afterloaded('sparse', sparseloaded)
+  > EOF
+  $ cat >> $HGRCPATH <<EOF
+  > [extensions]
+  > trap_sparse_refresh=$TESTTMP/trap_sparse_refresh.py
+  > EOF
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > [include]
+  > *.py
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import
+
+If an exception is raised during refresh, restore the existing rules again.
+
+  $ cat > $TESTTMP/rules_to_import <<EOF
+  > [exclude]
+  > *.html
+  > EOF
+  $ hg sparse --import-rules $TESTTMP/rules_to_import
+  abort: sparse._refresh called!
+  [255]
+  $ cat .hg/sparse
+  %include webpage.sparse
+  [include]
+  *.py
+  *.txt
+  [exclude]
+  *.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-merges.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,62 @@
+test merging things outside of the sparse checkout
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > EOF
+
+  $ echo foo > foo
+  $ echo bar > bar
+  $ hg add foo bar
+  $ hg commit -m initial
+
+  $ hg branch feature
+  marked working directory as branch feature
+  (branches are permanent and global, did you want a bookmark?)
+  $ echo bar2 >> bar
+  $ hg commit -m 'feature - bar2'
+
+  $ hg update -q default
+  $ hg sparse --exclude 'bar**'
+
+  $ hg merge feature
+  temporarily included 1 file(s) in the sparse checkout for merging
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  (branch merge, don't forget to commit)
+
+Verify bar was merged temporarily
+
+  $ ls
+  bar
+  foo
+  $ hg status
+  M bar
+
+Verify bar disappears automatically when the working copy becomes clean
+
+  $ hg commit -m "merged"
+  cleaned up 1 temporarily added file(s) from the sparse checkout
+  $ hg status
+  $ ls
+  foo
+
+  $ hg cat -r . bar
+  bar
+  bar2
+
+Test merging things outside of the sparse checkout that are not in the working
+copy
+
+  $ hg strip -q -r . --config extensions.strip=
+  $ hg up -q feature
+  $ touch branchonly
+  $ hg ci -Aqm 'add branchonly'
+
+  $ hg up -q default
+  $ hg sparse -X branchonly
+  $ hg merge feature
+  temporarily included 2 file(s) in the sparse checkout for merging
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  (branch merge, don't forget to commit)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-profiles.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,272 @@
+test sparse
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > purge=
+  > strip=
+  > rebase=
+  > EOF
+
+  $ echo a > index.html
+  $ echo x > data.py
+  $ echo z > readme.txt
+  $ cat > webpage.sparse <<EOF
+  > # frontend sparse profile
+  > [include]
+  > *.html
+  > EOF
+  $ cat > backend.sparse <<EOF
+  > # backend sparse profile
+  > [include]
+  > *.py
+  > EOF
+  $ hg ci -Aqm 'initial'
+
+  $ hg sparse --include '*.sparse'
+
+Verify enabling a single profile works
+
+  $ hg sparse --enable-profile webpage.sparse
+  $ ls
+  backend.sparse
+  index.html
+  webpage.sparse
+
+Verify enabling two profiles works
+
+  $ hg sparse --enable-profile backend.sparse
+  $ ls
+  backend.sparse
+  data.py
+  index.html
+  webpage.sparse
+
+Verify disabling a profile works
+
+  $ hg sparse --disable-profile webpage.sparse
+  $ ls
+  backend.sparse
+  data.py
+  webpage.sparse
+
+Verify that a profile is updated across multiple commits
+
+  $ cat > webpage.sparse <<EOF
+  > # frontend sparse profile
+  > [include]
+  > *.html
+  > EOF
+  $ cat > backend.sparse <<EOF
+  > # backend sparse profile
+  > [include]
+  > *.py
+  > *.txt
+  > EOF
+
+  $ echo foo >> data.py
+
+  $ hg ci -m 'edit profile'
+  $ ls
+  backend.sparse
+  data.py
+  readme.txt
+  webpage.sparse
+
+  $ hg up -q 0
+  $ ls
+  backend.sparse
+  data.py
+  webpage.sparse
+
+  $ hg up -q 1
+  $ ls
+  backend.sparse
+  data.py
+  readme.txt
+  webpage.sparse
+
+Introduce a conflicting .hgsparse change
+
+  $ hg up -q 0
+  $ cat > backend.sparse <<EOF
+  > # Different backend sparse profile
+  > [include]
+  > *.html
+  > EOF
+  $ echo bar >> data.py
+
+  $ hg ci -qAm "edit profile other"
+  $ ls
+  backend.sparse
+  index.html
+  webpage.sparse
+
+Verify conflicting merge pulls in the conflicting changes
+
+  $ hg merge 1
+  temporarily included 1 file(s) in the sparse checkout for merging
+  merging backend.sparse
+  merging data.py
+  warning: conflicts while merging backend.sparse! (edit, then use 'hg resolve --mark')
+  warning: conflicts while merging data.py! (edit, then use 'hg resolve --mark')
+  0 files updated, 0 files merged, 0 files removed, 2 files unresolved
+  use 'hg resolve' to retry unresolved file merges or 'hg update -C .' to abandon
+  [1]
+
+  $ rm *.orig
+  $ ls
+  backend.sparse
+  data.py
+  index.html
+  webpage.sparse
+
+Verify resolving the merge removes the temporarily unioned files
+
+  $ cat > backend.sparse <<EOF
+  > # backend sparse profile
+  > [include]
+  > *.html
+  > *.txt
+  > EOF
+  $ hg resolve -m backend.sparse
+
+  $ cat > data.py <<EOF
+  > x
+  > foo
+  > bar
+  > EOF
+  $ hg resolve -m data.py
+  (no more unresolved files)
+
+  $ hg ci -qAm "merge profiles"
+  $ ls
+  backend.sparse
+  index.html
+  readme.txt
+  webpage.sparse
+
+  $ hg cat -r . data.py
+  x
+  foo
+  bar
+
+Verify stripping refreshes dirstate
+
+  $ hg strip -q -r .
+  $ ls
+  backend.sparse
+  index.html
+  webpage.sparse
+
+Verify rebase conflicts pulls in the conflicting changes
+
+  $ hg up -q 1
+  $ ls
+  backend.sparse
+  data.py
+  readme.txt
+  webpage.sparse
+
+  $ hg rebase -d 2
+  rebasing 1:a2b1de640a62 "edit profile"
+  temporarily included 1 file(s) in the sparse checkout for merging
+  merging backend.sparse
+  merging data.py
+  warning: conflicts while merging backend.sparse! (edit, then use 'hg resolve --mark')
+  warning: conflicts while merging data.py! (edit, then use 'hg resolve --mark')
+  unresolved conflicts (see hg resolve, then hg rebase --continue)
+  [1]
+  $ rm *.orig
+  $ ls
+  backend.sparse
+  data.py
+  index.html
+  webpage.sparse
+
+Verify resolving conflict removes the temporary files
+
+  $ cat > backend.sparse <<EOF
+  > [include]
+  > *.html
+  > *.txt
+  > EOF
+  $ hg resolve -m backend.sparse
+
+  $ cat > data.py <<EOF
+  > x
+  > foo
+  > bar
+  > EOF
+  $ hg resolve -m data.py
+  (no more unresolved files)
+  continue: hg rebase --continue
+
+  $ hg rebase -q --continue
+  $ ls
+  backend.sparse
+  index.html
+  readme.txt
+  webpage.sparse
+
+  $ hg cat -r . data.py
+  x
+  foo
+  bar
+
+Test checking out a commit that does not contain the sparse profile. The
+warning message can be suppressed by setting missingwarning = false in
+[sparse] section of your config:
+
+  $ hg sparse --reset
+  $ hg rm *.sparse
+  $ hg commit -m "delete profiles"
+  $ hg up -q ".^"
+  $ hg sparse --enable-profile backend.sparse
+  $ ls
+  index.html
+  readme.txt
+  $ hg up tip | grep warning
+  warning: sparse profile 'backend.sparse' not found in rev bfcb76de99cc - ignoring it
+  [1]
+  $ ls
+  data.py
+  index.html
+  readme.txt
+  $ hg sparse --disable-profile backend.sparse | grep warning
+  warning: sparse profile 'backend.sparse' not found in rev bfcb76de99cc - ignoring it
+  [1]
+  $ cat >> .hg/hgrc <<EOF
+  > [sparse]
+  > missingwarning = false
+  > EOF
+  $ hg sparse --enable-profile backend.sparse
+
+  $ cd ..
+
+Test file permissions changing across a sparse profile change
+  $ hg init sparseperm
+  $ cd sparseperm
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > EOF
+  $ touch a b
+  $ cat > .hgsparse <<EOF
+  > a
+  > EOF
+  $ hg commit -Aqm 'initial'
+  $ chmod a+x b
+  $ hg commit -qm 'make executable'
+  $ cat >> .hgsparse <<EOF
+  > b
+  > EOF
+  $ hg commit -qm 'update profile'
+  $ hg up -q 0
+  $ hg sparse --enable-profile .hgsparse
+  $ hg up -q 2
+  $ ls -l b
+  -rwxr-xr-x* b (glob)
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse-verbose-json.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,82 @@
+test sparse with --verbose and -T json
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > strip=
+  > EOF
+
+  $ echo a > show
+  $ echo x > hide
+  $ hg ci -Aqm 'initial'
+
+  $ echo b > show
+  $ echo y > hide
+  $ echo aa > show2
+  $ echo xx > hide2
+  $ hg ci -Aqm 'two'
+
+Verify basic --include and --reset
+
+  $ hg up -q 0
+  $ hg sparse --include 'hide' -Tjson
+  [
+   {
+    "exclude_rules_added": 0,
+    "files_added": 0,
+    "files_conflicting": 0,
+    "files_dropped": 1,
+    "include_rules_added": 1,
+    "profiles_added": 0
+   }
+  ]
+  $ hg sparse --clear-rules
+  $ hg sparse --include 'hide' --verbose
+  removing show
+  Profile # change: 0
+  Include rule # change: 1
+  Exclude rule # change: 0
+
+  $ hg sparse --reset -Tjson
+  [
+   {
+    "exclude_rules_added": 0,
+    "files_added": 1,
+    "files_conflicting": 0,
+    "files_dropped": 0,
+    "include_rules_added": -1,
+    "profiles_added": 0
+   }
+  ]
+  $ hg sparse --include 'hide'
+  $ hg sparse --reset --verbose
+  getting show
+  Profile # change: 0
+  Include rule # change: -1
+  Exclude rule # change: 0
+
+Verifying that problematic files still allow us to see the deltas when forcing:
+
+  $ hg sparse --include 'show*'
+  $ touch hide
+  $ hg sparse --delete 'show*' --force -Tjson
+  pending changes to 'hide'
+  [
+   {
+    "exclude_rules_added": 0,
+    "files_added": 0,
+    "files_conflicting": 1,
+    "files_dropped": 0,
+    "include_rules_added": -1,
+    "profiles_added": 0
+   }
+  ]
+  $ hg sparse --include 'show*' --force
+  pending changes to 'hide'
+  $ hg sparse --delete 'show*' --force --verbose
+  pending changes to 'hide'
+  Profile # change: 0
+  Include rule # change: -1
+  Exclude rule # change: 0
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-sparse.t	Sat Jul 01 10:43:29 2017 -0700
@@ -0,0 +1,369 @@
+test sparse
+
+  $ hg init myrepo
+  $ cd myrepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > strip=
+  > EOF
+
+  $ echo a > show
+  $ echo x > hide
+  $ hg ci -Aqm 'initial'
+
+  $ echo b > show
+  $ echo y > hide
+  $ echo aa > show2
+  $ echo xx > hide2
+  $ hg ci -Aqm 'two'
+
+Verify basic --include
+
+  $ hg up -q 0
+  $ hg sparse --include 'hide'
+  $ ls
+  hide
+
+Absolute paths outside the repo should just be rejected
+
+  $ hg sparse --include /foo/bar
+  warning: paths cannot start with /, ignoring: ['/foo/bar']
+  $ hg sparse --include '$TESTTMP/myrepo/hide'
+
+  $ hg sparse --include '/root'
+  warning: paths cannot start with /, ignoring: ['/root']
+
+Verify commiting while sparse includes other files
+
+  $ echo z > hide
+  $ hg ci -Aqm 'edit hide'
+  $ ls
+  hide
+  $ hg manifest
+  hide
+  show
+
+Verify --reset brings files back
+
+  $ hg sparse --reset
+  $ ls
+  hide
+  show
+  $ cat hide
+  z
+  $ cat show
+  a
+
+Verify 'hg sparse' default output
+
+  $ hg up -q null
+  $ hg sparse --include 'show*'
+
+  $ hg sparse
+  [include]
+  show*
+  [exclude]
+
+
+Verify update only writes included files
+
+  $ hg up -q 0
+  $ ls
+  show
+
+  $ hg up -q 1
+  $ ls
+  show
+  show2
+
+Verify status only shows included files
+
+  $ touch hide
+  $ touch hide3
+  $ echo c > show
+  $ hg status
+  M show
+
+Adding an excluded file should fail
+
+  $ hg add hide3
+  abort: cannot add 'hide3' - it is outside the sparse checkout
+  (include file with `hg sparse --include <pattern>` or use `hg add -s <file>` to include file directory while adding)
+  [255]
+
+Verify deleting sparseness while a file has changes fails
+
+  $ hg sparse --delete 'show*'
+  pending changes to 'hide'
+  abort: cannot change sparseness due to pending changes (delete the files or use --force to bring them back dirty)
+  [255]
+
+Verify deleting sparseness with --force brings back files
+
+  $ hg sparse --delete -f 'show*'
+  pending changes to 'hide'
+  $ ls
+  hide
+  hide2
+  hide3
+  show
+  show2
+  $ hg st
+  M hide
+  M show
+  ? hide3
+
+Verify editing sparseness fails if pending changes
+
+  $ hg sparse --include 'show*'
+  pending changes to 'hide'
+  abort: could not update sparseness due to pending changes
+  [255]
+
+Verify adding sparseness hides files
+
+  $ hg sparse --exclude -f 'hide*'
+  pending changes to 'hide'
+  $ ls
+  hide
+  hide3
+  show
+  show2
+  $ hg st
+  M show
+
+  $ hg up -qC .
+  $ hg purge --all --config extensions.purge=
+  $ ls
+  show
+  show2
+
+Verify rebase temporarily includes excluded files
+
+  $ hg rebase -d 1 -r 2 --config extensions.rebase=
+  rebasing 2:b91df4f39e75 "edit hide" (tip)
+  temporarily included 1 file(s) in the sparse checkout for merging
+  merging hide
+  warning: conflicts while merging hide! (edit, then use 'hg resolve --mark')
+  unresolved conflicts (see hg resolve, then hg rebase --continue)
+  [1]
+
+  $ hg sparse
+  [include]
+
+  [exclude]
+  hide*
+
+  Temporarily Included Files (for merge/rebase):
+  hide
+
+  $ cat hide
+  <<<<<<< dest:   39278f7c08a9 - test: two
+  y
+  =======
+  z
+  >>>>>>> source: b91df4f39e75 - test: edit hide
+
+Verify aborting a rebase cleans up temporary files
+
+  $ hg rebase --abort --config extensions.rebase=
+  cleaned up 1 temporarily added file(s) from the sparse checkout
+  rebase aborted
+  $ rm hide.orig
+
+  $ ls
+  show
+  show2
+
+Verify merge fails if merging excluded files
+
+  $ hg up -q 1
+  $ hg merge -r 2
+  temporarily included 1 file(s) in the sparse checkout for merging
+  merging hide
+  warning: conflicts while merging hide! (edit, then use 'hg resolve --mark')
+  0 files updated, 0 files merged, 0 files removed, 1 files unresolved
+  use 'hg resolve' to retry unresolved file merges or 'hg update -C .' to abandon
+  [1]
+  $ hg sparse
+  [include]
+
+  [exclude]
+  hide*
+
+  Temporarily Included Files (for merge/rebase):
+  hide
+
+  $ hg up -C .
+  cleaned up 1 temporarily added file(s) from the sparse checkout
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg sparse
+  [include]
+
+  [exclude]
+  hide*
+
+
+Verify strip -k resets dirstate correctly
+
+  $ hg status
+  $ hg sparse
+  [include]
+
+  [exclude]
+  hide*
+
+  $ hg log -r . -T '{rev}\n' --stat
+  1
+   hide  |  2 +-
+   hide2 |  1 +
+   show  |  2 +-
+   show2 |  1 +
+   4 files changed, 4 insertions(+), 2 deletions(-)
+
+  $ hg strip -r . -k
+  saved backup bundle to $TESTTMP/myrepo/.hg/strip-backup/39278f7c08a9-ce59e002-backup.hg (glob)
+  $ hg status
+  M show
+  ? show2
+
+Verify rebase succeeds if all changed files are in sparse checkout
+
+  $ hg commit -Aqm "add show2"
+  $ hg rebase -d 1 --config extensions.rebase=
+  rebasing 2:bdde55290160 "add show2" (tip)
+  saved backup bundle to $TESTTMP/myrepo/.hg/strip-backup/bdde55290160-216ed9c6-backup.hg (glob)
+
+Verify log --sparse only shows commits that affect the sparse checkout
+
+  $ hg log -T '{rev} '
+  2 1 0  (no-eol)
+  $ hg log --sparse -T '{rev} '
+  2 0  (no-eol)
+
+Test status on a file in a subdir
+
+  $ mkdir -p dir1/dir2
+  $ touch dir1/dir2/file
+  $ hg sparse -I dir1/dir2
+  $ hg status
+  ? dir1/dir2/file
+
+Test that add -s adds dirs to sparse profile
+
+  $ hg sparse --reset
+  $ hg sparse --include empty
+  $ hg sparse
+  [include]
+  empty
+  [exclude]
+
+
+
+  $ mkdir add
+  $ touch add/foo
+  $ touch add/bar
+  $ hg add add/foo
+  abort: cannot add 'add/foo' - it is outside the sparse checkout
+  (include file with `hg sparse --include <pattern>` or use `hg add -s <file>` to include file directory while adding)
+  [255]
+  $ hg add -s add/foo
+  $ hg st
+  A add/foo
+  ? add/bar
+  $ hg sparse
+  [include]
+  add
+  empty
+  [exclude]
+
+
+  $ hg add -s add/*
+  add/foo already tracked!
+  $ hg st
+  A add/bar
+  A add/foo
+  $ hg sparse
+  [include]
+  add
+  empty
+  [exclude]
+
+
+
+  $ cd ..
+
+Test non-sparse repos work while sparse is loaded
+  $ hg init sparserepo
+  $ hg init nonsparserepo
+  $ cd sparserepo
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > sparse=
+  > EOF
+  $ cd ../nonsparserepo
+  $ echo x > x && hg add x && hg commit -qAm x
+  $ cd ../sparserepo
+  $ hg clone ../nonsparserepo ../nonsparserepo2
+  updating to branch default
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+Test debugrebuilddirstate
+  $ cd ../sparserepo
+  $ touch included
+  $ touch excluded
+  $ hg add included excluded
+  $ hg commit -m 'a commit' -q
+  $ cp .hg/dirstate ../dirstateboth
+  $ hg sparse -X excluded
+  $ cp ../dirstateboth .hg/dirstate
+  $ hg debugrebuilddirstate
+  $ hg debugdirstate
+  n   0         -1 unset               included
+
+Test debugdirstate --minimal where file is in the parent manifest but not the
+dirstate
+  $ hg sparse -X included
+  $ hg debugdirstate
+  $ cp .hg/dirstate ../dirstateallexcluded
+  $ hg sparse --reset
+  $ hg sparse -X excluded
+  $ cp ../dirstateallexcluded .hg/dirstate
+  $ touch includedadded
+  $ hg add includedadded
+  $ hg debugdirstate --nodates
+  a   0         -1 unset               includedadded
+  $ hg debugrebuilddirstate --minimal
+  $ hg debugdirstate --nodates
+  n   0         -1 unset               included
+  a   0         -1 * includedadded (glob)
+
+Test debugdirstate --minimal where a file is not in parent manifest
+but in the dirstate. This should take into account excluded files in the
+manifest
+  $ cp ../dirstateboth .hg/dirstate
+  $ touch includedadded
+  $ hg add includedadded
+  $ touch excludednomanifest
+  $ hg add excludednomanifest
+  $ cp .hg/dirstate ../moreexcluded
+  $ hg forget excludednomanifest
+  $ rm excludednomanifest
+  $ hg sparse -X excludednomanifest
+  $ cp ../moreexcluded .hg/dirstate
+  $ hg manifest
+  excluded
+  included
+We have files in the dirstate that are included and excluded. Some are in the
+manifest and some are not.
+  $ hg debugdirstate --nodates
+  n 644          0 * excluded (glob)
+  a   0         -1 * excludednomanifest (glob)
+  n 644          0 * included (glob)
+  a   0         -1 * includedadded (glob)
+  $ hg debugrebuilddirstate --minimal
+  $ hg debugdirstate --nodates
+  n 644          0 * included (glob)
+  a   0         -1 * includedadded (glob)
+