hgext/remotefilelog/remotefilelog.py

tests: show the files fields of changelogs for many merges

I don't think there's coverage for many of the subtle cases, and I
found it hard to understand what the code is doing by reading it.  The
test takes 40s to run on a laptop, or 9s with --chg.

I have yet to find a description of what the files field is supposed
to be for merges. I thought it could be one of:
1. the files added/modified/removed relative to p1 (wouldn't seem
  useful, but `hg diff -c -r mergerev` has this behavior)
2. the files with filelog nodes not in either parent (i.e., what is
needed to create a bundle out of a commit)
3. the files added/removed/modified files by merge itself [1]

It's clearly not 1, because file contents merges are symmetric. It's
clearly not 2 because removed files and exec bit changes are
listed. It's also not 3 but I think it's intended to be 3 and the
differences are bugs.

Assuming 3, the test shows that, for merges, the list of files both
overapproximates and underapproximates. All the cases involve file
changes not in the filelog but in the manifest (existence of file
at revision, exec bit and file vs symlink).

I didn't look at all underapproximations, but they looked minor. The
two overapproximations are problematic though because they both cause
potentially long lists of files when merging cleanly.


[1] even what it means for the merge commit itself to change a file is
not completely trivial. A file in the merge being the same as in one
of the parent is too lax as it would consider that merges change
nothing when they revert all the changes done on one side. The
criteria used in the test and in the next commit for "merge didn't
touch a file" is:
- the parents and the merge all have the same file
- or, one parent didn't touch the file and the other parent contains
  the same file as the merge

Differential Revision: https://phab.mercurial-scm.org/D6612

# remotefilelog.py - filelog implementation where filelog history is stored
#                    remotely
#
# Copyright 2013 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import

import collections
import os

from mercurial.node import (
    bin,
    nullid,
    wdirfilenodeids,
    wdirid,
)
from mercurial.i18n import _
from mercurial import (
    ancestor,
    error,
    mdiff,
    revlog,
)
from mercurial.utils import storageutil

from . import (
    constants,
    fileserverclient,
    shallowutil,
)

class remotefilelognodemap(object):
    def __init__(self, filename, store):
        self._filename = filename
        self._store = store

    def __contains__(self, node):
        missing = self._store.getmissing([(self._filename, node)])
        return not bool(missing)

    def __get__(self, node):
        if node not in self:
            raise KeyError(node)
        return node

class remotefilelog(object):

    _generaldelta = True

    def __init__(self, opener, path, repo):
        self.opener = opener
        self.filename = path
        self.repo = repo
        self.nodemap = remotefilelognodemap(self.filename, repo.contentstore)

        self.version = 1

    def read(self, node):
        """returns the file contents at this node"""
        t = self.revision(node)
        if not t.startswith('\1\n'):
            return t
        s = t.index('\1\n', 2)
        return t[s + 2:]

    def add(self, text, meta, transaction, linknode, p1=None, p2=None):
        # hash with the metadata, like in vanilla filelogs
        hashtext = shallowutil.createrevlogtext(text, meta.get('copy'),
                                                meta.get('copyrev'))
        node = storageutil.hashrevisionsha1(hashtext, p1, p2)
        return self.addrevision(hashtext, transaction, linknode, p1, p2,
                                node=node)

    def _createfileblob(self, text, meta, flags, p1, p2, node, linknode):
        # text passed to "_createfileblob" does not include filelog metadata
        header = shallowutil.buildfileblobheader(len(text), flags)
        data = "%s\0%s" % (header, text)

        realp1 = p1
        copyfrom = ""
        if meta and 'copy' in meta:
            copyfrom = meta['copy']
            realp1 = bin(meta['copyrev'])

        data += "%s%s%s%s%s\0" % (node, realp1, p2, linknode, copyfrom)

        visited = set()

        pancestors = {}
        queue = []
        if realp1 != nullid:
            p1flog = self
            if copyfrom:
                p1flog = remotefilelog(self.opener, copyfrom, self.repo)

            pancestors.update(p1flog.ancestormap(realp1))
            queue.append(realp1)
            visited.add(realp1)
        if p2 != nullid:
            pancestors.update(self.ancestormap(p2))
            queue.append(p2)
            visited.add(p2)

        ancestortext = ""

        # add the ancestors in topological order
        while queue:
            c = queue.pop(0)
            pa1, pa2, ancestorlinknode, pacopyfrom = pancestors[c]

            pacopyfrom = pacopyfrom or ''
            ancestortext += "%s%s%s%s%s\0" % (
                c, pa1, pa2, ancestorlinknode, pacopyfrom)

            if pa1 != nullid and pa1 not in visited:
                queue.append(pa1)
                visited.add(pa1)
            if pa2 != nullid and pa2 not in visited:
                queue.append(pa2)
                visited.add(pa2)

        data += ancestortext

        return data

    def addrevision(self, text, transaction, linknode, p1, p2, cachedelta=None,
                    node=None, flags=revlog.REVIDX_DEFAULT_FLAGS):
        # text passed to "addrevision" includes hg filelog metadata header
        if node is None:
            node = storageutil.hashrevisionsha1(text, p1, p2)

        meta, metaoffset = storageutil.parsemeta(text)
        rawtext, validatehash = self._processflags(text, flags, 'write')
        return self.addrawrevision(rawtext, transaction, linknode, p1, p2,
                                   node, flags, cachedelta,
                                   _metatuple=(meta, metaoffset))

    def addrawrevision(self, rawtext, transaction, linknode, p1, p2, node,
                       flags, cachedelta=None, _metatuple=None):
        if _metatuple:
            # _metatuple: used by "addrevision" internally by remotefilelog
            # meta was parsed confidently
            meta, metaoffset = _metatuple
        else:
            # not from self.addrevision, but something else (repo._filecommit)
            # calls addrawrevision directly. remotefilelog needs to get and
            # strip filelog metadata.
            # we don't have confidence about whether rawtext contains filelog
            # metadata or not (flag processor could replace it), so we just
            # parse it as best-effort.
            # in LFS (flags != 0)'s case, the best way is to call LFS code to
            # get the meta information, instead of storageutil.parsemeta.
            meta, metaoffset = storageutil.parsemeta(rawtext)
        if flags != 0:
            # when flags != 0, be conservative and do not mangle rawtext, since
            # a read flag processor expects the text not being mangled at all.
            metaoffset = 0
        if metaoffset:
            # remotefilelog fileblob stores copy metadata in its ancestortext,
            # not its main blob. so we need to remove filelog metadata
            # (containing copy information) from text.
            blobtext = rawtext[metaoffset:]
        else:
            blobtext = rawtext
        data = self._createfileblob(blobtext, meta, flags, p1, p2, node,
                                    linknode)
        self.repo.contentstore.addremotefilelognode(self.filename, node, data)

        return node

    def renamed(self, node):
        ancestors = self.repo.metadatastore.getancestors(self.filename, node)
        p1, p2, linknode, copyfrom = ancestors[node]
        if copyfrom:
            return (copyfrom, p1)

        return False

    def size(self, node):
        """return the size of a given revision"""
        return len(self.read(node))

    rawsize = size

    def cmp(self, node, text):
        """compare text with a given file revision

        returns True if text is different than what is stored.
        """

        if node == nullid:
            return True

        nodetext = self.read(node)
        return nodetext != text

    def __nonzero__(self):
        return True

    __bool__ = __nonzero__

    def __len__(self):
        if self.filename == '.hgtags':
            # The length of .hgtags is used to fast path tag checking.
            # remotefilelog doesn't support .hgtags since the entire .hgtags
            # history is needed.  Use the excludepattern setting to make
            # .hgtags a normal filelog.
            return 0

        raise RuntimeError("len not supported")

    def empty(self):
        return False

    def flags(self, node):
        if isinstance(node, int):
            raise error.ProgrammingError(
                'remotefilelog does not accept integer rev for flags')
        store = self.repo.contentstore
        return store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)

    def parents(self, node):
        if node == nullid:
            return nullid, nullid

        ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
        p1, p2, linknode, copyfrom = ancestormap[node]
        if copyfrom:
            p1 = nullid

        return p1, p2

    def parentrevs(self, rev):
        # TODO(augie): this is a node and should be a rev, but for now
        # nothing in core seems to actually break.
        return self.parents(rev)

    def linknode(self, node):
        ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
        p1, p2, linknode, copyfrom = ancestormap[node]
        return linknode

    def linkrev(self, node):
        return self.repo.unfiltered().changelog.rev(self.linknode(node))

    def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
                      assumehaveparentrevisions=False, deltaprevious=False,
                      deltamode=None):
        # we don't use any of these parameters here
        del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
        del deltamode
        prevnode = None
        for node in nodes:
            p1, p2 = self.parents(node)
            if prevnode is None:
                basenode = prevnode = p1
            if basenode == node:
                basenode = nullid
            if basenode != nullid:
                revision = None
                delta = self.revdiff(basenode, node)
            else:
                revision = self.revision(node, raw=True)
                delta = None
            yield revlog.revlogrevisiondelta(
                node=node,
                p1node=p1,
                p2node=p2,
                linknode=self.linknode(node),
                basenode=basenode,
                flags=self.flags(node),
                baserevisionsize=None,
                revision=revision,
                delta=delta,
                )

    def revdiff(self, node1, node2):
        return mdiff.textdiff(self.revision(node1, raw=True),
                              self.revision(node2, raw=True))

    def lookup(self, node):
        if len(node) == 40:
            node = bin(node)
        if len(node) != 20:
            raise error.LookupError(node, self.filename,
                                    _('invalid lookup input'))

        return node

    def rev(self, node):
        # This is a hack to make TortoiseHG work.
        return node

    def node(self, rev):
        # This is a hack.
        if isinstance(rev, int):
            raise error.ProgrammingError(
                'remotefilelog does not convert integer rev to node')
        return rev

    def revision(self, node, raw=False):
        """returns the revlog contents at this node.
        this includes the meta data traditionally included in file revlogs.
        this is generally only used for bundling and communicating with vanilla
        hg clients.
        """
        if node == nullid:
            return ""
        if len(node) != 20:
            raise error.LookupError(node, self.filename,
                                    _('invalid revision input'))
        if node == wdirid or node in wdirfilenodeids:
            raise error.WdirUnsupported

        store = self.repo.contentstore
        rawtext = store.get(self.filename, node)
        if raw:
            return rawtext
        flags = store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)
        if flags == 0:
            return rawtext
        text, verifyhash = self._processflags(rawtext, flags, 'read')
        return text

    def _processflags(self, text, flags, operation, raw=False):
        # mostly copied from hg/mercurial/revlog.py
        validatehash = True
        orderedflags = revlog.REVIDX_FLAGS_ORDER
        if operation == 'write':
            orderedflags = reversed(orderedflags)
        for flag in orderedflags:
            if flag & flags:
                vhash = True
                if flag not in revlog._flagprocessors:
                    message = _("missing processor for flag '%#x'") % (flag)
                    raise revlog.RevlogError(message)
                readfunc, writefunc, rawfunc = revlog._flagprocessors[flag]
                if raw:
                    vhash = rawfunc(self, text)
                elif operation == 'read':
                    text, vhash = readfunc(self, text)
                elif operation == 'write':
                    text, vhash = writefunc(self, text)
                validatehash = validatehash and vhash
        return text, validatehash

    def _read(self, id):
        """reads the raw file blob from disk, cache, or server"""
        fileservice = self.repo.fileservice
        localcache = fileservice.localcache
        cachekey = fileserverclient.getcachekey(self.repo.name, self.filename,
                                                id)
        try:
            return localcache.read(cachekey)
        except KeyError:
            pass

        localkey = fileserverclient.getlocalkey(self.filename, id)
        localpath = os.path.join(self.localpath, localkey)
        try:
            return shallowutil.readfile(localpath)
        except IOError:
            pass

        fileservice.prefetch([(self.filename, id)])
        try:
            return localcache.read(cachekey)
        except KeyError:
            pass

        raise error.LookupError(id, self.filename, _('no node'))

    def ancestormap(self, node):
        return self.repo.metadatastore.getancestors(self.filename, node)

    def ancestor(self, a, b):
        if a == nullid or b == nullid:
            return nullid

        revmap, parentfunc = self._buildrevgraph(a, b)
        nodemap = dict(((v, k) for (k, v) in revmap.iteritems()))

        ancs = ancestor.ancestors(parentfunc, revmap[a], revmap[b])
        if ancs:
            # choose a consistent winner when there's a tie
            return min(map(nodemap.__getitem__, ancs))
        return nullid

    def commonancestorsheads(self, a, b):
        """calculate all the heads of the common ancestors of nodes a and b"""

        if a == nullid or b == nullid:
            return nullid

        revmap, parentfunc = self._buildrevgraph(a, b)
        nodemap = dict(((v, k) for (k, v) in revmap.iteritems()))

        ancs = ancestor.commonancestorsheads(parentfunc, revmap[a], revmap[b])
        return map(nodemap.__getitem__, ancs)

    def _buildrevgraph(self, a, b):
        """Builds a numeric revision graph for the given two nodes.
        Returns a node->rev map and a rev->[revs] parent function.
        """
        amap = self.ancestormap(a)
        bmap = self.ancestormap(b)

        # Union the two maps
        parentsmap = collections.defaultdict(list)
        allparents = set()
        for mapping in (amap, bmap):
            for node, pdata in mapping.iteritems():
                parents = parentsmap[node]
                p1, p2, linknode, copyfrom = pdata
                # Don't follow renames (copyfrom).
                # remotefilectx.ancestor does that.
                if p1 != nullid and not copyfrom:
                    parents.append(p1)
                    allparents.add(p1)
                if p2 != nullid:
                    parents.append(p2)
                    allparents.add(p2)

        # Breadth first traversal to build linkrev graph
        parentrevs = collections.defaultdict(list)
        revmap = {}
        queue = collections.deque(((None, n) for n in parentsmap
                 if n not in allparents))
        while queue:
            prevrev, current = queue.pop()
            if current in revmap:
                if prevrev:
                    parentrevs[prevrev].append(revmap[current])
                continue

            # Assign linkrevs in reverse order, so start at
            # len(parentsmap) and work backwards.
            currentrev = len(parentsmap) - len(revmap) - 1
            revmap[current] = currentrev

            if prevrev:
                parentrevs[prevrev].append(currentrev)

            for parent in parentsmap.get(current):
                queue.appendleft((currentrev, parent))

        return revmap, parentrevs.__getitem__

    def strip(self, minlink, transaction):
        pass

    # misc unused things
    def files(self):
        return []

    def checksize(self):
        return 0, 0