comparison hgext/largefiles/lfutil.py @ 43077:687b865b95ad

formatting: byteify all mercurial/ and hgext/ string literals

Done with

    python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py')
    black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**')

# skip-blame mass-reformatting only

Differential Revision: https://phab.mercurial-scm.org/D6972

43076:2372284d9457

    sparse,
    util,
    vfs as vfsmod,
)

shortname = '.hglf'
shortnameslash = shortname + '/'
longname = 'largefiles'

# -- Private worker functions ------------------------------------------


def getminsize(ui, assumelfiles, opt, default=10):
    lfsize = opt
    if not lfsize and assumelfiles:
        lfsize = ui.config(longname, 'minsize', default=default)
    if lfsize:
        try:
            lfsize = float(lfsize)
        except ValueError:
            raise error.Abort(
                _('largefiles: size must be number (not %s)\n') % lfsize
            )
    if lfsize is None:
        raise error.Abort(_('minimum size for largefiles must be specified'))
    return lfsize


def link(src, dest):
    """Try to create hardlink - if that fails, efficiently make a copy."""
    util.makedirs(os.path.dirname(dest))
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        with open(src, 'rb') as srcf, util.atomictempfile(dest) as dstf:
            for chunk in util.filechunkiter(srcf):
                dstf.write(chunk)
        os.chmod(dest, os.stat(src).st_mode)

    return os.path.join(_usercachedir(ui), hash)


def _usercachedir(ui, name=longname):
    '''Return the location of the "global" largefiles cache.'''
    path = ui.configpath(name, 'usercache')
    if path:
        return path
    if pycompat.iswindows:
        appdata = encoding.environ.get(
            'LOCALAPPDATA', encoding.environ.get('APPDATA')
        )
        if appdata:
            return os.path.join(appdata, name)
    elif pycompat.isdarwin:
        home = encoding.environ.get('HOME')
        if home:
            return os.path.join(home, 'Library', 'Caches', name)
    elif pycompat.isposix:
        path = encoding.environ.get('XDG_CACHE_HOME')
        if path:
            return os.path.join(path, name)
        home = encoding.environ.get('HOME')
        if home:
            return os.path.join(home, '.cache', name)
    else:
        raise error.Abort(_('unknown operating system: %s\n') % pycompat.osname)
    raise error.Abort(_('unknown %s usercache location') % name)


def inusercache(ui, hash):
    path = usercachepath(ui, hash)
    return os.path.exists(path)

    '''Return store path of the largefile with the specified hash.
    As a side effect, the file might be linked from user cache.
    Return None if the file can't be found locally.'''
    path, exists = findstorepath(repo, hash)
    if exists:
        repo.ui.note(_('found %s in store\n') % hash)
        return path
    elif inusercache(repo.ui, hash):
        repo.ui.note(_('found %s in system cache\n') % hash)
        path = storepath(repo, hash)
        link(usercachepath(repo.ui, hash), path)
        return path
    return None

    )

    # If the largefiles dirstate does not exist, populate and create
    # it. This ensures that we create it on the first meaningful
    # largefiles operation in a new clone.
    if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')):
        matcher = getstandinmatcher(repo)
        standins = repo.dirstate.walk(
            matcher, subrepos=[], unknown=False, ignored=False
        )

            lfdirstate.normallookup(lfile)
    return lfdirstate


def lfdirstatestatus(lfdirstate, repo):
    pctx = repo['.']
    match = matchmod.always()
    unsure, s = lfdirstate.status(
        match, subrepos=[], ignored=False, clean=False, unknown=False
    )
    modified, clean = s.modified, s.clean

    # ignore unknown files in working directory
    return [
        splitstandin(f)
        for f in repo[rev].walk(matcher)
        if rev is not None or repo.dirstate[f] != '?'
    ]


def instore(repo, hash, forcelocal=False):
    '''Return true if a largefile with the given hash exists in the store'''

    if path is None:
        return False
    wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
    # The write may fail before the file is fully written, but we
    # don't use atomic writes in the working copy.
    with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd:
        gothash = copyandhash(util.filechunkiter(srcfd), destfd)
    if gothash != hash:
        repo.ui.warn(
            _('%s: data corruption in %s with hash %s\n')
            % (filename, path, gothash)
        )
        wvfs.unlink(filename)
        return False
    return True

        return
    if wvfs.exists(file):
        copytostoreabsolute(repo, wvfs.join(file), hash)
    else:
        repo.ui.warn(
            _("%s: largefile %s not available from local store\n")
            % (file, hash)
        )


def copyalltostore(repo, node):

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        with open(file, 'rb') as srcf:
            with util.atomictempfile(
                storepath(repo, hash), createmode=repo.store.createmode
            ) as dstf:
                for chunk in util.filechunkiter(srcf):
                    dstf.write(chunk)

def splitstandin(filename):
    # Split on / because that's what dirstate always uses, even on Windows.
    # Change local separator to / first just in case we are passed filenames
    # from an external source (like the command line).
    bits = util.pconvert(filename).split('/', 1)
    if len(bits) == 2 and bits[0] == shortname:
        return bits[1]
    else:
        return None

    if repo.wvfs.exists(lfile):
        hash = hashfile(file)
        executable = getexecutable(file)
        writestandin(repo, standin, hash, executable)
    else:
        raise error.Abort(_('%s: file not found!') % lfile)


def readasstandin(fctx):
    '''read hex hash from given filectx of standin file

    return fctx.data().strip()


def writestandin(repo, standin, hash, executable):
    '''write hash to <repo.root>/<standin>'''
    repo.wwrite(standin, hash + '\n', executable and 'x' or '')


def copyandhash(instream, outfile):
    '''Read bytes from instream (iterable) and write them to outfile,
    computing the SHA-1 hash of the data along the way. Return the hash.'''
    hasher = hashlib.sha1('')
    for data in instream:
        hasher.update(data)
        outfile.write(data)
    return hex(hasher.digest())


def hashfile(file):
    if not os.path.exists(file):
        return ''
    with open(file, 'rb') as fd:
        return hexsha1(fd)


def getexecutable(filename):
    mode = os.stat(filename).st_mode

    )


def urljoin(first, second, *arg):
    def join(left, right):
        if not left.endswith('/'):
            left += '/'
        if right.startswith('/'):
            right = right[1:]
        return left + right

    url = join(first, second)
    for a in arg:

        h.update(chunk)
    return hex(h.digest())


def httpsendfile(ui, filename):
    return httpconnection.httpsendfile(ui, filename, 'rb')


def unixpath(path):
    '''Return a version of path normalized for use with the lfdirstate.'''
    return util.pconvert(os.path.normpath(path))


def islfilesrepo(repo):
    '''Return true if the repo is a largefile repo.'''
    if 'largefiles' in repo.requirements and any(
        shortnameslash in f[0] for f in repo.store.datafiles()
    ):
        return True

    return any(openlfdirstate(repo.ui, repo, False))

    lfstandin = standin(lfile)
    if lfstandin in repo.dirstate:
        stat = repo.dirstate._map[lfstandin]
        state, mtime = stat[0], stat[3]
    else:
        state, mtime = '?', -1
    if state == 'n':
        if normallookup or mtime < 0 or not repo.wvfs.exists(lfile):
            # state 'n' doesn't ensure 'clean' in this case
            lfdirstate.normallookup(lfile)
        else:
            lfdirstate.normal(lfile)
    elif state == 'm':
        lfdirstate.normallookup(lfile)
    elif state == 'r':
        lfdirstate.remove(lfile)
    elif state == 'a':
        lfdirstate.add(lfile)
    elif state == '?':
        lfdirstate.drop(lfile)


def markcommitted(orig, ctx, node):
    repo = ctx.repo()

def getlfilestoupload(repo, missing, addfunc):
    makeprogress = repo.ui.makeprogress
    with makeprogress(
        _('finding outgoing largefiles'),
        unit=_('revisions'),
        total=len(missing),
    ) as progress:
        for i, n in enumerate(missing):
            progress.update(i)
            parents = [p for p in repo[n].parents() if p != node.nullid]

    # asked to commit them, so sooner or later we're going to
    # refresh the standins.  Might as well leave them refreshed.
    lfdirstate = openlfdirstate(ui, repo)
    for fstandin in standins:
        lfile = splitstandin(fstandin)
        if lfdirstate[lfile] != 'r':
            updatestandin(repo, lfile, fstandin)

    # Cook up a new matcher that only matches regular files or
    # standins corresponding to the big files requested by the
    # user.  Have to modify _files to prevent commit() from

        # For largefiles, only one of the normal and standin should be
        # committed (except if one of them is a remove).  In the case of a
        # standin removal, drop the normal file if it is unknown to dirstate.
        # Thus, skip plain largefile names but keep the standin.
        if f in lfiles or fstandin in standins:
            if repo.dirstate[fstandin] != 'r':
                if repo.dirstate[f] != 'r':
                    continue
            elif repo.dirstate[f] == '?':
                continue

        actualfiles.append(f)
    match._files = actualfiles

    ``repo._lfstatuswriters`` as "default" writer function.

    Otherwise, this returns the function to always write out (or
    ignore if ``not forcibly``) status.
    '''
    if forcibly is None and util.safehasattr(repo, '_largefilesenabled'):
        return repo._lfstatuswriters[-1]
    else:
        if forcibly:
            return ui.status  # forcibly WRITE OUT
        else:

43077:687b865b95ad

    sparse,
    util,
    vfs as vfsmod,
)

shortname = b'.hglf'
shortnameslash = shortname + b'/'
longname = b'largefiles'

# -- Private worker functions ------------------------------------------


def getminsize(ui, assumelfiles, opt, default=10):
    lfsize = opt
    if not lfsize and assumelfiles:
        lfsize = ui.config(longname, b'minsize', default=default)
    if lfsize:
        try:
            lfsize = float(lfsize)
        except ValueError:
            raise error.Abort(
                _(b'largefiles: size must be number (not %s)\n') % lfsize
            )
    if lfsize is None:
        raise error.Abort(_(b'minimum size for largefiles must be specified'))
    return lfsize


def link(src, dest):
    """Try to create hardlink - if that fails, efficiently make a copy."""
    util.makedirs(os.path.dirname(dest))
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        with open(src, b'rb') as srcf, util.atomictempfile(dest) as dstf:
            for chunk in util.filechunkiter(srcf):
                dstf.write(chunk)
        os.chmod(dest, os.stat(src).st_mode)

    return os.path.join(_usercachedir(ui), hash)


def _usercachedir(ui, name=longname):
    '''Return the location of the "global" largefiles cache.'''
    path = ui.configpath(name, b'usercache')
    if path:
        return path
    if pycompat.iswindows:
        appdata = encoding.environ.get(
            b'LOCALAPPDATA', encoding.environ.get(b'APPDATA')
        )
        if appdata:
            return os.path.join(appdata, name)
    elif pycompat.isdarwin:
        home = encoding.environ.get(b'HOME')
        if home:
            return os.path.join(home, b'Library', b'Caches', name)
    elif pycompat.isposix:
        path = encoding.environ.get(b'XDG_CACHE_HOME')
        if path:
            return os.path.join(path, name)
        home = encoding.environ.get(b'HOME')
        if home:
            return os.path.join(home, b'.cache', name)
    else:
        raise error.Abort(
            _(b'unknown operating system: %s\n') % pycompat.osname
        )
    raise error.Abort(_(b'unknown %s usercache location') % name)


def inusercache(ui, hash):
    path = usercachepath(ui, hash)
    return os.path.exists(path)

    '''Return store path of the largefile with the specified hash.
    As a side effect, the file might be linked from user cache.
    Return None if the file can't be found locally.'''
    path, exists = findstorepath(repo, hash)
    if exists:
        repo.ui.note(_(b'found %s in store\n') % hash)
        return path
    elif inusercache(repo.ui, hash):
        repo.ui.note(_(b'found %s in system cache\n') % hash)
        path = storepath(repo, hash)
        link(usercachepath(repo.ui, hash), path)
        return path
    return None

    )

    # If the largefiles dirstate does not exist, populate and create
    # it. This ensures that we create it on the first meaningful
    # largefiles operation in a new clone.
    if create and not vfs.exists(vfs.join(lfstoredir, b'dirstate')):
        matcher = getstandinmatcher(repo)
        standins = repo.dirstate.walk(
            matcher, subrepos=[], unknown=False, ignored=False
        )

            lfdirstate.normallookup(lfile)
    return lfdirstate


def lfdirstatestatus(lfdirstate, repo):
    pctx = repo[b'.']
    match = matchmod.always()
    unsure, s = lfdirstate.status(
        match, subrepos=[], ignored=False, clean=False, unknown=False
    )
    modified, clean = s.modified, s.clean

    # ignore unknown files in working directory
    return [
        splitstandin(f)
        for f in repo[rev].walk(matcher)
        if rev is not None or repo.dirstate[f] != b'?'
    ]


def instore(repo, hash, forcelocal=False):
    '''Return true if a largefile with the given hash exists in the store'''

    if path is None:
        return False
    wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
    # The write may fail before the file is fully written, but we
    # don't use atomic writes in the working copy.
    with open(path, b'rb') as srcfd, wvfs(filename, b'wb') as destfd:
        gothash = copyandhash(util.filechunkiter(srcfd), destfd)
    if gothash != hash:
        repo.ui.warn(
            _(b'%s: data corruption in %s with hash %s\n')
            % (filename, path, gothash)
        )
        wvfs.unlink(filename)
        return False
    return True

        return
    if wvfs.exists(file):
        copytostoreabsolute(repo, wvfs.join(file), hash)
    else:
        repo.ui.warn(
            _(b"%s: largefile %s not available from local store\n")
            % (file, hash)
        )


def copyalltostore(repo, node):

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        with open(file, b'rb') as srcf:
            with util.atomictempfile(
                storepath(repo, hash), createmode=repo.store.createmode
            ) as dstf:
                for chunk in util.filechunkiter(srcf):
                    dstf.write(chunk)

def splitstandin(filename):
    # Split on / because that's what dirstate always uses, even on Windows.
    # Change local separator to / first just in case we are passed filenames
    # from an external source (like the command line).
    bits = util.pconvert(filename).split(b'/', 1)
    if len(bits) == 2 and bits[0] == shortname:
        return bits[1]
    else:
        return None

    if repo.wvfs.exists(lfile):
        hash = hashfile(file)
        executable = getexecutable(file)
        writestandin(repo, standin, hash, executable)
    else:
        raise error.Abort(_(b'%s: file not found!') % lfile)


def readasstandin(fctx):
    '''read hex hash from given filectx of standin file

    return fctx.data().strip()


def writestandin(repo, standin, hash, executable):
    '''write hash to <repo.root>/<standin>'''
    repo.wwrite(standin, hash + b'\n', executable and b'x' or b'')


def copyandhash(instream, outfile):
    '''Read bytes from instream (iterable) and write them to outfile,
    computing the SHA-1 hash of the data along the way. Return the hash.'''
    hasher = hashlib.sha1(b'')
    for data in instream:
        hasher.update(data)
        outfile.write(data)
    return hex(hasher.digest())


def hashfile(file):
    if not os.path.exists(file):
        return b''
    with open(file, b'rb') as fd:
        return hexsha1(fd)


def getexecutable(filename):
    mode = os.stat(filename).st_mode

    )


def urljoin(first, second, *arg):
    def join(left, right):
        if not left.endswith(b'/'):
            left += b'/'
        if right.startswith(b'/'):
            right = right[1:]
        return left + right

    url = join(first, second)
    for a in arg:

        h.update(chunk)
    return hex(h.digest())


def httpsendfile(ui, filename):
    return httpconnection.httpsendfile(ui, filename, b'rb')


def unixpath(path):
    '''Return a version of path normalized for use with the lfdirstate.'''
    return util.pconvert(os.path.normpath(path))


def islfilesrepo(repo):
    '''Return true if the repo is a largefile repo.'''
    if b'largefiles' in repo.requirements and any(
        shortnameslash in f[0] for f in repo.store.datafiles()
    ):
        return True

    return any(openlfdirstate(repo.ui, repo, False))

    lfstandin = standin(lfile)
    if lfstandin in repo.dirstate:
        stat = repo.dirstate._map[lfstandin]
        state, mtime = stat[0], stat[3]
    else:
        state, mtime = b'?', -1
    if state == b'n':
        if normallookup or mtime < 0 or not repo.wvfs.exists(lfile):
            # state 'n' doesn't ensure 'clean' in this case
            lfdirstate.normallookup(lfile)
        else:
            lfdirstate.normal(lfile)
    elif state == b'm':
        lfdirstate.normallookup(lfile)
    elif state == b'r':
        lfdirstate.remove(lfile)
    elif state == b'a':
        lfdirstate.add(lfile)
    elif state == b'?':
        lfdirstate.drop(lfile)


def markcommitted(orig, ctx, node):
    repo = ctx.repo()

def getlfilestoupload(repo, missing, addfunc):
    makeprogress = repo.ui.makeprogress
    with makeprogress(
        _(b'finding outgoing largefiles'),
        unit=_(b'revisions'),
        total=len(missing),
    ) as progress:
        for i, n in enumerate(missing):
            progress.update(i)
            parents = [p for p in repo[n].parents() if p != node.nullid]

    # asked to commit them, so sooner or later we're going to
    # refresh the standins.  Might as well leave them refreshed.
    lfdirstate = openlfdirstate(ui, repo)
    for fstandin in standins:
        lfile = splitstandin(fstandin)
        if lfdirstate[lfile] != b'r':
            updatestandin(repo, lfile, fstandin)

    # Cook up a new matcher that only matches regular files or
    # standins corresponding to the big files requested by the
    # user.  Have to modify _files to prevent commit() from

        # For largefiles, only one of the normal and standin should be
        # committed (except if one of them is a remove).  In the case of a
        # standin removal, drop the normal file if it is unknown to dirstate.
        # Thus, skip plain largefile names but keep the standin.
        if f in lfiles or fstandin in standins:
            if repo.dirstate[fstandin] != b'r':
                if repo.dirstate[f] != b'r':
                    continue
            elif repo.dirstate[f] == b'?':
                continue

        actualfiles.append(f)
    match._files = actualfiles

    ``repo._lfstatuswriters`` as "default" writer function.

    Otherwise, this returns the function to always write out (or
    ignore if ``not forcibly``) status.
    '''
    if forcibly is None and util.safehasattr(repo, b'_largefilesenabled'):
        return repo._lfstatuswriters[-1]
    else:
        if forcibly:
            return ui.status  # forcibly WRITE OUT
        else: