view hgext/censor.py @ 27396:64c584070fc7

run-tests: show scheduling with --showchannels

This gives one line of output per second with one column per -j level
that allows analyzing test scheduling problems. First 24 seconds of
output at -j 30 looks like this:

  0                                                                .
  1  = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =   s.
  2  c c o c r l g r s s = c p = c h c a h c g c h c b c c l l c   ss
  3  h o b o e a e u u u c o a h o e o c g o l h g h u o = a o =   s
  4  e n s n b r n n b b   m t g n l n l w n o e w e n n e r g i   .
  5  c t o = a g d - r r = m c w v p v . e v g c e c d v x g . m
  6  k r l r s e o t e e b a h e e . e . b e . k b k l e t e . p
  7  - i e e e f c e p p u n b b r . r . - r . - - - e r e f . o   .
  8  p b t v - i . s o o n d o d t . t . c t . c s = 2 t n i . r
  9  y - e s c l . t - . d - m i - . - . o - . o y r - - s l . t
 10  3 p - e h e . s s . l t b r s . s . m s . d m e f s i e . .
 11  - e c t e s . . v . e e . . v . v . m v . e r n o v o s . .
 12  c r h . c - . . n . 2 m . . n . n . a n . . e a r n n . . .
 13  o f e . k u . . . . - p . . - . - . n - . . v m m - . . . .
 14  m . c . - p . . . . e l . . s . m . d s . . . e a e . . . .
 15  p . k . r d . . . . x a . . i . o . s o . . . - t n . . . .
 16  a . h . e a . . . . c t . . n . v . . u . . . m . c . . . .
 17  t . e . s t . . . . h e . . k . e . . r . . . e . o . . . .
 18  . . a . t e . . . . a . . . . . . . . c . . . r . d . . . .
 19  . . d . o . . . . . n . . . . . . . . e . . . g . i . . . .
 20  . . s . r . . . . . g . . . . . . . . . . . . e . n . . . .
 21  . . . . e . . . . . e . . . . . . . . . . . . 2 . g . . . .
 22  . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
 23  . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .   .
 24  . . . . . . . . . . . . . . . . . . . . . . . . . = . . . .   ^C

Test names read off vertically, beginning with '='. Idle time (not
shown) appears as blank space.

# Copyright (C) 2015 - Mike Edgar <[e-mail]>
#
# This extension enables removal of file content at a given revision,
# rewriting the data/metadata of successive revisions to preserve revision log
# integrity.

"""erase file content at a given revision

The censor command instructs Mercurial to erase all content of a file at a given
revision *without updating the changeset hash.* This allows existing history to
remain valid while preventing future clones/pulls from receiving the erased
data.

Typical uses for censor are due to security or legal requirements, including::

 * Passwords, private keys, cryptographic material
 * Licensed data/code/libraries for which the license has expired
 * Personally Identifiable Information or other private data

Censored nodes can interrupt mercurial's typical operation whenever the excised
data needs to be materialized. Some commands, like ``hg cat``/``hg revert``,
simply fail when asked to produce censored data. Others, like ``hg verify`` and
``hg update``, must be capable of tolerating censored data to continue to
function in a meaningful way. Such commands only tolerate censored file
revisions if they are allowed by the "censor.policy=ignore" config option.
"""

from mercurial.node import short
from mercurial import cmdutil, error, filelog, revlog, scmutil, util
from mercurial.i18n import _
from mercurial import lock as lockmod

cmdtable = {}
command = cmdutil.command(cmdtable)
# Note for extension authors: ONLY specify testedwith = 'internal' for
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or
# leave the attribute unspecified.
testedwith = 'internal'

@command('censor',
    [('r', 'rev', '', _('censor file from specified revision'), _('REV')),
     ('t', 'tombstone', '', _('replacement tombstone data'), _('TEXT'))],
    _('-r REV [-t TEXT] [FILE]'))
def censor(ui, repo, path, rev='', tombstone='', **opts):
    wlock = lock = None
    try:
        wlock = repo.wlock()
        lock = repo.lock()
        return _docensor(ui, repo, path, rev, tombstone, **opts)
    finally:
        lockmod.release(lock, wlock)

def _docensor(ui, repo, path, rev='', tombstone='', **opts):
    if not path:
        raise error.Abort(_('must specify file path to censor'))
    if not rev:
        raise error.Abort(_('must specify revision to censor'))

    wctx = repo[None]

    m = scmutil.match(wctx, (path,))
    if m.anypats() or len(m.files()) != 1:
        raise error.Abort(_('can only specify an explicit filename'))
    path = m.files()[0]
    flog = repo.file(path)
    if not len(flog):
        raise error.Abort(_('cannot censor file with no history'))

    rev = scmutil.revsingle(repo, rev, rev).rev()
    try:
        ctx = repo[rev]
    except KeyError:
        raise error.Abort(_('invalid revision identifier %s') % rev)

    try:
        fctx = ctx.filectx(path)
    except error.LookupError:
        raise error.Abort(_('file does not exist at revision %s') % rev)

    fnode = fctx.filenode()
    headctxs = [repo[c] for c in repo.heads()]
    heads = [c for c in headctxs if path in c and c.filenode(path) == fnode]
    if heads:
        headlist = ', '.join([short(c.node()) for c in heads])
        raise error.Abort(_('cannot censor file in heads (%s)') % headlist,
            hint=_('clean/delete and commit first'))

    wp = wctx.parents()
    if ctx.node() in [p.node() for p in wp]:
        raise error.Abort(_('cannot censor working directory'),
            hint=_('clean/delete/update first'))

    flogv = flog.version & 0xFFFF
    if flogv != revlog.REVLOGNG:
        raise error.Abort(
            _('censor does not support revlog version %d') % (flogv,))

    tombstone = filelog.packmeta({"censored": tombstone}, "")

    crev = fctx.filerev()

    if len(tombstone) > flog.rawsize(crev):
        raise error.Abort(_(
            'censor tombstone must be no longer than censored data'))

    # Using two files instead of one makes it easy to rewrite entry-by-entry
    idxread = repo.svfs(flog.indexfile, 'r')
    idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
    if flog.version & revlog.REVLOGNGINLINEDATA:
        dataread, datawrite = idxread, idxwrite
    else:
        dataread = repo.svfs(flog.datafile, 'r')
        datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)

    # Copy all revlog data up to the entry to be censored.
    rio = revlog.revlogio()
    offset = flog.start(crev)

    for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
        idxwrite.write(chunk)
    for chunk in util.filechunkiter(dataread, limit=offset):
        datawrite.write(chunk)

    def rewriteindex(r, newoffs, newdata=None):
        """Rewrite the index entry with a new data offset and optional new data.

        The newdata argument, if given, is a tuple of three positive integers:
        (new compressed, new uncompressed, added flag bits).
        """
        offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
        flags = revlog.gettype(offlags)
        if newdata:
            comp, uncomp, nflags = newdata
            flags |= nflags
        offlags = revlog.offset_type(newoffs, flags)
        e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
        idxwrite.write(rio.packentry(e, None, flog.version, r))
        idxread.seek(rio.size, 1)

    def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
        """Write the given full text to the filelog with the given data offset.

        Returns:
            The integer number of data bytes written, for tracking data offsets.
        """
        flag, compdata = flog.compress(data)
        newcomp = len(flag) + len(compdata)
        rewriteindex(r, offs, (newcomp, len(data), nflags))
        datawrite.write(flag)
        datawrite.write(compdata)
        dataread.seek(flog.length(r), 1)
        return newcomp

    # Rewrite censored revlog entry with (padded) tombstone data.
    pad = ' ' * (flog.rawsize(crev) - len(tombstone))
    offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)

    # Rewrite all following filelog revisions fixing up offsets and deltas.
    for srev in xrange(crev + 1, len(flog)):
        if crev in flog.parentrevs(srev):
            # Immediate children of censored node must be re-added as fulltext.
            try:
                revdata = flog.revision(srev)
            except error.CensoredNodeError as e:
                revdata = e.tombstone
            dlen = rewrite(srev, offset, revdata)
        else:
            # Copy any other revision data verbatim after fixing up the offset.
            rewriteindex(srev, offset)
            dlen = flog.length(srev)
            for chunk in util.filechunkiter(dataread, limit=dlen):
                datawrite.write(chunk)
        offset += dlen

    idxread.close()
    idxwrite.close()
    if dataread is not idxread:
        dataread.close()
        datawrite.close()