view mercurial/revlogutils/flagutil.py @ 45095:8e04607023e5

procutil: ensure that procutil.std{out,err}.write() writes all bytes

Python 3 offers different kind of streams and it’s not guaranteed for all of
them that calling write() writes all bytes.

When Python is started in unbuffered mode, sys.std{out,err}.buffer are
instances of io.FileIO, whose write() can write less bytes for
platform-specific reasons (e.g. Linux has a 0x7ffff000 bytes maximum and could
write less if interrupted by a signal; when writing to Windows consoles, it’s
limited to 32767 bytes to avoid the "not enough space" error). This can lead to
silent loss of data, both when using sys.std{out,err}.buffer (which may in fact
not be a buffered stream) and when using the text streams sys.std{out,err}
(I’ve created a CPython bug report for that:
https://bugs.python.org/issue41221).

Python may fix the problem at some point. For now, we implement our own wrapper
for procutil.std{out,err} that calls the raw stream’s write() method until all
bytes have been written. We don’t use sys.std{out,err} for larger writes, so I
think it’s not worth the effort to patch them.

# flagutils.py - code to deal with revlog flags and their processors
#
# Copyright 2016 Remi Chaintron <[e-mail]>
# Copyright 2016-2019 Pierre-Yves David <[e-mail]>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

from ..i18n import _

from .constants import (
    REVIDX_DEFAULT_FLAGS,
    REVIDX_ELLIPSIS,
    REVIDX_EXTSTORED,
    REVIDX_FLAGS_ORDER,
    REVIDX_ISCENSORED,
    REVIDX_RAWTEXT_CHANGING_FLAGS,
    REVIDX_SIDEDATA,
)

from .. import error, util

# blanked usage of all the name to prevent pyflakes constraints
# We need these name available in the module for extensions.
REVIDX_ISCENSORED
REVIDX_ELLIPSIS
REVIDX_EXTSTORED
REVIDX_SIDEDATA
REVIDX_DEFAULT_FLAGS
REVIDX_FLAGS_ORDER
REVIDX_RAWTEXT_CHANGING_FLAGS

REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER)

# Store flag processors (cf. 'addflagprocessor()' to register)
flagprocessors = {
    REVIDX_ISCENSORED: None,
}


def addflagprocessor(flag, processor):
    """Register a flag processor on a revision data flag.

    Invariant:
    - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
      and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
    - Only one flag processor can be registered on a specific flag.
    - flagprocessors must be 3-tuples of functions (read, write, raw) with the
      following signatures:
          - (read)  f(self, rawtext) -> text, bool
          - (write) f(self, text) -> rawtext, bool
          - (raw)   f(self, rawtext) -> bool
      "text" is presented to the user. "rawtext" is stored in revlog data, not
      directly visible to the user.
      The boolean returned by these transforms is used to determine whether
      the returned text can be used for hash integrity checking. For example,
      if "write" returns False, then "text" is used to generate hash. If
      "write" returns True, that basically means "rawtext" returned by "write"
      should be used to generate hash. Usually, "write" and "read" return
      different booleans. And "raw" returns a same boolean as "write".

      Note: The 'raw' transform is used for changegroup generation and in some
      debug commands. In this case the transform only indicates whether the
      contents can be used for hash integrity checks.
    """
    insertflagprocessor(flag, processor, flagprocessors)


def insertflagprocessor(flag, processor, flagprocessors):
    if not flag & REVIDX_KNOWN_FLAGS:
        msg = _(b"cannot register processor on unknown flag '%#x'.") % flag
        raise error.ProgrammingError(msg)
    if flag not in REVIDX_FLAGS_ORDER:
        msg = _(b"flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % flag
        raise error.ProgrammingError(msg)
    if flag in flagprocessors:
        msg = _(b"cannot register multiple processors on flag '%#x'.") % flag
        raise error.Abort(msg)
    flagprocessors[flag] = processor


def processflagswrite(revlog, text, flags, sidedata):
    """Inspect revision data flags and applies write transformations defined
    by registered flag processors.

    ``text`` - the revision data to process
    ``flags`` - the revision flags

    This method processes the flags in the order (or reverse order if
    ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
    flag processors registered for present flags. The order of flags defined
    in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.

    Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
    processed text and ``validatehash`` is a bool indicating whether the
    returned text should be checked for hash integrity.
    """
    return _processflagsfunc(revlog, text, flags, b'write', sidedata=sidedata)[
        :2
    ]


def processflagsread(revlog, text, flags):
    """Inspect revision data flags and applies read transformations defined
    by registered flag processors.

    ``text`` - the revision data to process
    ``flags`` - the revision flags
    ``raw`` - an optional argument describing if the raw transform should be
    applied.

    This method processes the flags in the order (or reverse order if
    ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
    flag processors registered for present flags. The order of flags defined
    in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.

    Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
    processed text and ``validatehash`` is a bool indicating whether the
    returned text should be checked for hash integrity.
    """
    return _processflagsfunc(revlog, text, flags, b'read')


def processflagsraw(revlog, text, flags):
    """Inspect revision data flags to check is the content hash should be
    validated.

    ``text`` - the revision data to process
    ``flags`` - the revision flags

    This method processes the flags in the order (or reverse order if
    ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
    flag processors registered for present flags. The order of flags defined
    in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.

    Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
    processed text and ``validatehash`` is a bool indicating whether the
    returned text should be checked for hash integrity.
    """
    return _processflagsfunc(revlog, text, flags, b'raw')[1]


def _processflagsfunc(revlog, text, flags, operation, sidedata=None):
    """internal function to process flag on a revlog

    This function is private to this module, code should never needs to call it
    directly."""
    # fast path: no flag processors will run
    if flags == 0:
        return text, True, {}
    if operation not in (b'read', b'write', b'raw'):
        raise error.ProgrammingError(_(b"invalid '%s' operation") % operation)
    # Check all flags are known.
    if flags & ~REVIDX_KNOWN_FLAGS:
        raise revlog._flagserrorclass(
            _(b"incompatible revision flag '%#x'")
            % (flags & ~REVIDX_KNOWN_FLAGS)
        )
    validatehash = True
    # Depending on the operation (read or write), the order might be
    # reversed due to non-commutative transforms.
    orderedflags = REVIDX_FLAGS_ORDER
    if operation == b'write':
        orderedflags = reversed(orderedflags)

    outsidedata = {}
    for flag in orderedflags:
        # If a flagprocessor has been registered for a known flag, apply the
        # related operation transform and update result tuple.
        if flag & flags:
            vhash = True

            if flag not in revlog._flagprocessors:
                message = _(b"missing processor for flag '%#x'") % flag
                raise revlog._flagserrorclass(message)

            processor = revlog._flagprocessors[flag]
            if processor is not None:
                readtransform, writetransform, rawtransform = processor

                if operation == b'raw':
                    vhash = rawtransform(revlog, text)
                elif operation == b'read':
                    text, vhash, s = readtransform(revlog, text)
                    outsidedata.update(s)
                else:  # write operation
                    text, vhash = writetransform(revlog, text, sidedata)
            validatehash = validatehash and vhash

    return text, validatehash, outsidedata