comparison mercurial/store.py @ 46895:6085b7f1536d

store: also return some information about the type of file `walk` found

We start returning of 4th information in the `store.walk` return tuple: the type of the file. This will make it easier for caller to determine which kind of file they are looking at. This should especically help with the `upgrade-repo` code that has to do a lot of fragile index's file name comparison.

Differential Revision: https://phab.mercurial-scm.org/D10315

46894:fe34c75f62ab

    b'phaseroots',
    b'obsstore',
    b'requires',
]

REVLOG_FILES_EXT = (b'.i', b'.d', b'.n', b'.nd')


def isrevlog(f, kind, st):
    if kind != stat.S_IFREG:
        return False
    return f.endswith(REVLOG_FILES_EXT)


class basicstore(object):
    '''base class for local repository stores'''

            readdir = self.rawvfs.readdir
            while visit:
                p = visit.pop()
                for f, kind, st in readdir(p, stat=True):
                    fp = p + b'/' + f
                    if isrevlog(f, kind, st):
                        n = util.pconvert(fp[striplen:])
                        l.append((decodedir(n), n, st.st_size))
                    elif kind == stat.S_IFDIR and recurse:
                        visit.append(fp)
        l.sort()
        return l

    def manifestlog(self, repo, storenarrowmatch):
        rootstore = manifest.manifestrevlog(repo.nodeconstants, self.vfs)
        return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch)

    def datafiles(self, matcher=None):
        return self._walk(b'data', True) + self._walk(b'meta', True)

    def topfiles(self):
        # yield manifest before changelog
        return reversed(self._walk(b'', False))

    def walk(self, matcher=None):
        """return file related to data storage (ie: revlogs)

        yields (unencoded, encoded, size)

        if a matcher is passed, storage files of only those tracked paths
        are passed with matches the matcher
        """
        # yield data files first

        self.rawvfs = vfs
        self.vfs = vfsmod.filtervfs(vfs, encodefilename)
        self.opener = self.vfs

    def datafiles(self, matcher=None):
        for a, b, size in super(encodedstore, self).datafiles():
            try:
                a = decodefilename(a)
            except KeyError:
                a = None
            if a is not None and not _matchtrackedpath(a, matcher):
                continue
            yield a, b, size

    def join(self, f):
        return self.path + b'/' + encodefilename(f)

    def copylist(self):

        for f in sorted(self.fncache):
            if not _matchtrackedpath(f, matcher):
                continue
            ef = self.encode(f)
            try:
                yield f, ef, self.getsize(ef)
            except OSError as err:
                if err.errno != errno.ENOENT:
                    raise

    def copylist(self):

46895:6085b7f1536d

    b'phaseroots',
    b'obsstore',
    b'requires',
]

REVLOG_FILES_MAIN_EXT = (b'.i', b'i.tmpcensored')
REVLOG_FILES_OTHER_EXT = (b'.d', b'.n', b'.nd', b'd.tmpcensored')


def is_revlog(f, kind, st):
    if kind != stat.S_IFREG:
        return None
    return revlog_type(f)


def revlog_type(f):
    if f.endswith(REVLOG_FILES_MAIN_EXT):
        return FILEFLAGS_REVLOG_MAIN
    elif f.endswith(REVLOG_FILES_OTHER_EXT):
        return FILETYPE_FILELOG_OTHER


# the file is part of changelog data
FILEFLAGS_CHANGELOG = 1 << 13
# the file is part of manifest data
FILEFLAGS_MANIFESTLOG = 1 << 12
# the file is part of filelog data
FILEFLAGS_FILELOG = 1 << 11
# file that are not directly part of a revlog
FILEFLAGS_OTHER = 1 << 10

# the main entry point for a revlog
FILEFLAGS_REVLOG_MAIN = 1 << 1
# a secondary file for a revlog
FILEFLAGS_REVLOG_OTHER = 1 << 0

FILETYPE_CHANGELOG_MAIN = FILEFLAGS_CHANGELOG | FILEFLAGS_REVLOG_MAIN
FILETYPE_CHANGELOG_OTHER = FILEFLAGS_CHANGELOG | FILEFLAGS_REVLOG_OTHER
FILETYPE_MANIFESTLOG_MAIN = FILEFLAGS_MANIFESTLOG | FILEFLAGS_REVLOG_MAIN
FILETYPE_MANIFESTLOG_OTHER = FILEFLAGS_MANIFESTLOG | FILEFLAGS_REVLOG_OTHER
FILETYPE_FILELOG_MAIN = FILEFLAGS_FILELOG | FILEFLAGS_REVLOG_MAIN
FILETYPE_FILELOG_OTHER = FILEFLAGS_FILELOG | FILEFLAGS_REVLOG_OTHER
FILETYPE_OTHER = FILEFLAGS_OTHER


class basicstore(object):
    '''base class for local repository stores'''

            readdir = self.rawvfs.readdir
            while visit:
                p = visit.pop()
                for f, kind, st in readdir(p, stat=True):
                    fp = p + b'/' + f
                    rl_type = is_revlog(f, kind, st)
                    if rl_type is not None:
                        n = util.pconvert(fp[striplen:])
                        l.append((rl_type, decodedir(n), n, st.st_size))
                    elif kind == stat.S_IFDIR and recurse:
                        visit.append(fp)
        l.sort()
        return l

    def manifestlog(self, repo, storenarrowmatch):
        rootstore = manifest.manifestrevlog(repo.nodeconstants, self.vfs)
        return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch)

    def datafiles(self, matcher=None):
        files = self._walk(b'data', True) + self._walk(b'meta', True)
        for (t, u, e, s) in files:
            yield (FILEFLAGS_FILELOG | t, u, e, s)

    def topfiles(self):
        # yield manifest before changelog
        files = reversed(self._walk(b'', False))
        for (t, u, e, s) in files:
            if u.startswith(b'00changelog'):
                yield (FILEFLAGS_CHANGELOG | t, u, e, s)
            elif u.startswith(b'00manifest'):
                yield (FILEFLAGS_MANIFESTLOG | t, u, e, s)
            else:
                yield (FILETYPE_OTHER | t, u, e, s)

    def walk(self, matcher=None):
        """return file related to data storage (ie: revlogs)

        yields (file_type, unencoded, encoded, size)

        if a matcher is passed, storage files of only those tracked paths
        are passed with matches the matcher
        """
        # yield data files first

        self.rawvfs = vfs
        self.vfs = vfsmod.filtervfs(vfs, encodefilename)
        self.opener = self.vfs

    def datafiles(self, matcher=None):
        for t, a, b, size in super(encodedstore, self).datafiles():
            try:
                a = decodefilename(a)
            except KeyError:
                a = None
            if a is not None and not _matchtrackedpath(a, matcher):
                continue
            yield t, a, b, size

    def join(self, f):
        return self.path + b'/' + encodefilename(f)

    def copylist(self):

        for f in sorted(self.fncache):
            if not _matchtrackedpath(f, matcher):
                continue
            ef = self.encode(f)
            try:
                t = revlog_type(f)
                t |= FILEFLAGS_FILELOG
                yield t, f, ef, self.getsize(ef)
            except OSError as err:
                if err.errno != errno.ENOENT:
                    raise

    def copylist(self):