Mercurial > hg
view hgext/releasenotes.py @ 33499:0407a51b9d8c
codemod: register core configitems using a script
This is done by a script [2] using RedBaron [1], a tool designed for doing
code refactoring. All "default" values are decided by the script and are
strongly consistent with the existing code.
There are 2 changes done manually to fix tests:
[warn] mercurial/exchange.py: experimental.bundle2-output-capture: default needs manual removal
[warn] mercurial/localrepo.py: experimental.hook-track-tags: default needs manual removal
Since RedBaron is not confident about how to indent things [2].
[1]: https://github.com/PyCQA/redbaron
[2]: https://github.com/PyCQA/redbaron/issues/100
[3]:
#!/usr/bin/env python
# codemod_configitems.py - codemod tool to fill configitems
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import, print_function
import os
import sys
import redbaron
def readpath(path):
with open(path) as f:
return f.read()
def writepath(path, content):
with open(path, 'w') as f:
f.write(content)
_configmethods = {'config', 'configbool', 'configint', 'configbytes',
'configlist', 'configdate'}
def extractstring(rnode):
"""get the string from a RedBaron string or call_argument node"""
while rnode.type != 'string':
rnode = rnode.value
return rnode.value[1:-1] # unquote, "'str'" -> "str"
def uiconfigitems(red):
"""match *.ui.config* pattern, yield (node, method, args, section, name)"""
for node in red.find_all('atomtrailers'):
entry = None
try:
obj = node[-3].value
method = node[-2].value
args = node[-1]
section = args[0].value
name = args[1].value
if (obj in ('ui', 'self') and method in _configmethods
and section.type == 'string' and name.type == 'string'):
entry = (node, method, args, extractstring(section),
extractstring(name))
except Exception:
pass
else:
if entry:
yield entry
def coreconfigitems(red):
"""match coreconfigitem(...) pattern, yield (node, args, section, name)"""
for node in red.find_all('atomtrailers'):
entry = None
try:
args = node[1]
section = args[0].value
name = args[1].value
if (node[0].value == 'coreconfigitem' and section.type == 'string'
and name.type == 'string'):
entry = (node, args, extractstring(section),
extractstring(name))
except Exception:
pass
else:
if entry:
yield entry
def registercoreconfig(cfgred, section, name, defaultrepr):
"""insert coreconfigitem to cfgred AST
section and name are plain string, defaultrepr is a string
"""
# find a place to insert the "coreconfigitem" item
entries = list(coreconfigitems(cfgred))
for node, args, nodesection, nodename in reversed(entries):
if (nodesection, nodename) < (section, name):
# insert after this entry
node.insert_after(
'coreconfigitem(%r, %r,\n'
' default=%s,\n'
')' % (section, name, defaultrepr))
return
def main(argv):
if not argv:
print('Usage: codemod_configitems.py FILES\n'
'For example, FILES could be "{hgext,mercurial}/*/**.py"')
dirname = os.path.dirname
reporoot = dirname(dirname(dirname(os.path.abspath(__file__))))
# register configitems to this destination
cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py')
cfgred = redbaron.RedBaron(readpath(cfgpath))
# state about what to do
registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred))
toregister = {} # {(section, name): defaultrepr}
coreconfigs = set() # {(section, name)}, whether it's used in core
# first loop: scan all files before taking any action
for i, path in enumerate(argv):
print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
iscore = ('mercurial' in path) and ('hgext' not in path)
red = redbaron.RedBaron(readpath(path))
# find all repo.ui.config* and ui.config* calls, and collect their
# section, name and default value information.
for node, method, args, section, name in uiconfigitems(red):
if section == 'web':
# [web] section has some weirdness, ignore them for now
continue
defaultrepr = None
key = (section, name)
if len(args) == 2:
if key in registered:
continue
if method == 'configlist':
defaultrepr = 'list'
elif method == 'configbool':
defaultrepr = 'False'
else:
defaultrepr = 'None'
elif len(args) >= 3 and (args[2].target is None or
args[2].target.value == 'default'):
# try to understand the "default" value
dnode = args[2].value
if dnode.type == 'name':
if dnode.value in {'None', 'True', 'False'}:
defaultrepr = dnode.value
elif dnode.type == 'string':
defaultrepr = repr(dnode.value[1:-1])
elif dnode.type in ('int', 'float'):
defaultrepr = dnode.value
# inconsistent default
if key in toregister and toregister[key] != defaultrepr:
defaultrepr = None
# interesting to rewrite
if key not in registered:
if defaultrepr is None:
print('[note] %s: %s.%s: unsupported default'
% (path, section, name))
registered.add(key) # skip checking it again
else:
toregister[key] = defaultrepr
if iscore:
coreconfigs.add(key)
# second loop: rewrite files given "toregister" result
for path in argv:
# reconstruct redbaron - trade CPU for memory
red = redbaron.RedBaron(readpath(path))
changed = False
for node, method, args, section, name in uiconfigitems(red):
key = (section, name)
defaultrepr = toregister.get(key)
if defaultrepr is None or key not in coreconfigs:
continue
if len(args) >= 3 and (args[2].target is None or
args[2].target.value == 'default'):
try:
del args[2]
changed = True
except Exception:
# redbaron fails to do the rewrite due to indentation
# see https://github.com/PyCQA/redbaron/issues/100
print('[warn] %s: %s.%s: default needs manual removal'
% (path, section, name))
if key not in registered:
print('registering %s.%s' % (section, name))
registercoreconfig(cfgred, section, name, defaultrepr)
registered.add(key)
if changed:
print('updating %s' % path)
writepath(path, red.dumps())
if toregister:
print('updating configitems.py')
writepath(cfgpath, cfgred.dumps())
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
author | Jun Wu <quark@fb.com> |
---|---|
date | Fri, 14 Jul 2017 14:22:40 -0700 |
parents | 5814db57941c |
children | 9a944e908ecf |
line wrap: on
line source
# Copyright 2017-present Gregory Szorc <gregory.szorc@gmail.com> # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. """generate release notes from commit messages (EXPERIMENTAL) It is common to maintain files detailing changes in a project between releases. Maintaining these files can be difficult and time consuming. The :hg:`releasenotes` command provided by this extension makes the process simpler by automating it. """ from __future__ import absolute_import import errno import re import sys import textwrap from mercurial.i18n import _ from mercurial import ( error, minirst, registrar, scmutil, ) cmdtable = {} command = registrar.command(cmdtable) # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # be specifying the version(s) of Mercurial they are tested with, or # leave the attribute unspecified. testedwith = 'ships-with-hg-core' DEFAULT_SECTIONS = [ ('feature', _('New Features')), ('bc', _('Backwards Compatibility Changes')), ('fix', _('Bug Fixes')), ('perf', _('Performance Improvements')), ('api', _('API Changes')), ] RE_DIRECTIVE = re.compile('^\.\. ([a-zA-Z0-9_]+)::\s*([^$]+)?$') BULLET_SECTION = _('Other Changes') class parsedreleasenotes(object): def __init__(self): self.sections = {} def __contains__(self, section): return section in self.sections def __iter__(self): return iter(sorted(self.sections)) def addtitleditem(self, section, title, paragraphs): """Add a titled release note entry.""" self.sections.setdefault(section, ([], [])) self.sections[section][0].append((title, paragraphs)) def addnontitleditem(self, section, paragraphs): """Adds a non-titled release note entry. Will be rendered as a bullet point. """ self.sections.setdefault(section, ([], [])) self.sections[section][1].append(paragraphs) def titledforsection(self, section): """Returns titled entries in a section. Returns a list of (title, paragraphs) tuples describing sub-sections. """ return self.sections.get(section, ([], []))[0] def nontitledforsection(self, section): """Returns non-titled, bulleted paragraphs in a section.""" return self.sections.get(section, ([], []))[1] def hastitledinsection(self, section, title): return any(t[0] == title for t in self.titledforsection(section)) def merge(self, ui, other): """Merge another instance into this one. This is used to combine multiple sources of release notes together. """ for section in other: for title, paragraphs in other.titledforsection(section): if self.hastitledinsection(section, title): # TODO prompt for resolution if different and running in # interactive mode. ui.write(_('%s already exists in %s section; ignoring\n') % (title, section)) continue # TODO perform similarity comparison and try to match against # existing. self.addtitleditem(section, title, paragraphs) for paragraphs in other.nontitledforsection(section): if paragraphs in self.nontitledforsection(section): continue # TODO perform similarily comparison and try to match against # existing. self.addnontitleditem(section, paragraphs) class releasenotessections(object): def __init__(self, ui): # TODO support defining custom sections from config. self._sections = list(DEFAULT_SECTIONS) def __iter__(self): return iter(self._sections) def names(self): return [t[0] for t in self._sections] def sectionfromtitle(self, title): for name, value in self._sections: if value == title: return name return None def parsenotesfromrevisions(repo, directives, revs): notes = parsedreleasenotes() for rev in revs: ctx = repo[rev] blocks, pruned = minirst.parse(ctx.description(), admonitions=directives) for i, block in enumerate(blocks): if block['type'] != 'admonition': continue directive = block['admonitiontitle'] title = block['lines'][0].strip() if block['lines'] else None if i + 1 == len(blocks): raise error.Abort(_('release notes directive %s lacks content') % directive) # Now search ahead and find all paragraphs attached to this # admonition. paragraphs = [] for j in range(i + 1, len(blocks)): pblock = blocks[j] # Margin blocks may appear between paragraphs. Ignore them. if pblock['type'] == 'margin': continue if pblock['type'] != 'paragraph': raise error.Abort(_('unexpected block in release notes ' 'directive %s') % directive) if pblock['indent'] > 0: paragraphs.append(pblock['lines']) else: break # TODO consider using title as paragraph for more concise notes. if not paragraphs: raise error.Abort(_('could not find content for release note ' '%s') % directive) if title: notes.addtitleditem(directive, title, paragraphs) else: notes.addnontitleditem(directive, paragraphs) return notes def parsereleasenotesfile(sections, text): """Parse text content containing generated release notes.""" notes = parsedreleasenotes() blocks = minirst.parse(text)[0] def gatherparagraphsbullets(offset, title=False): notefragment = [] for i in range(offset + 1, len(blocks)): block = blocks[i] if block['type'] == 'margin': continue elif block['type'] == 'section': break elif block['type'] == 'bullet': if block['indent'] != 0: raise error.Abort(_('indented bullet lists not supported')) if title: lines = [l[1:].strip() for l in block['lines']] notefragment.append(lines) continue else: lines = [[l[1:].strip() for l in block['lines']]] for block in blocks[i + 1:]: if block['type'] in ('bullet', 'section'): break if block['type'] == 'paragraph': lines.append(block['lines']) notefragment.append(lines) continue elif block['type'] != 'paragraph': raise error.Abort(_('unexpected block type in release notes: ' '%s') % block['type']) if title: notefragment.append(block['lines']) return notefragment currentsection = None for i, block in enumerate(blocks): if block['type'] != 'section': continue title = block['lines'][0] # TODO the parsing around paragraphs and bullet points needs some # work. if block['underline'] == '=': # main section name = sections.sectionfromtitle(title) if not name: raise error.Abort(_('unknown release notes section: %s') % title) currentsection = name bullet_points = gatherparagraphsbullets(i) if bullet_points: for para in bullet_points: notes.addnontitleditem(currentsection, para) elif block['underline'] == '-': # sub-section if title == BULLET_SECTION: bullet_points = gatherparagraphsbullets(i) for para in bullet_points: notes.addnontitleditem(currentsection, para) else: paragraphs = gatherparagraphsbullets(i, True) notes.addtitleditem(currentsection, title, paragraphs) else: raise error.Abort(_('unsupported section type for %s') % title) return notes def serializenotes(sections, notes): """Serialize release notes from parsed fragments and notes. This function essentially takes the output of ``parsenotesfromrevisions()`` and ``parserelnotesfile()`` and produces output combining the 2. """ lines = [] for sectionname, sectiontitle in sections: if sectionname not in notes: continue lines.append(sectiontitle) lines.append('=' * len(sectiontitle)) lines.append('') # First pass to emit sub-sections. for title, paragraphs in notes.titledforsection(sectionname): lines.append(title) lines.append('-' * len(title)) lines.append('') wrapper = textwrap.TextWrapper(width=78) for i, para in enumerate(paragraphs): if i: lines.append('') lines.extend(wrapper.wrap(' '.join(para))) lines.append('') # Second pass to emit bullet list items. # If the section has titled and non-titled items, we can't # simply emit the bullet list because it would appear to come # from the last title/section. So, we emit a new sub-section # for the non-titled items. nontitled = notes.nontitledforsection(sectionname) if notes.titledforsection(sectionname) and nontitled: # TODO make configurable. lines.append(BULLET_SECTION) lines.append('-' * len(BULLET_SECTION)) lines.append('') for paragraphs in nontitled: wrapper = textwrap.TextWrapper(initial_indent='* ', subsequent_indent=' ', width=78) lines.extend(wrapper.wrap(' '.join(paragraphs[0]))) wrapper = textwrap.TextWrapper(initial_indent=' ', subsequent_indent=' ', width=78) for para in paragraphs[1:]: lines.append('') lines.extend(wrapper.wrap(' '.join(para))) lines.append('') if lines[-1]: lines.append('') return '\n'.join(lines) @command('releasenotes', [('r', 'rev', '', _('revisions to process for release notes'), _('REV'))], _('[-r REV] FILE')) def releasenotes(ui, repo, file_, rev=None): """parse release notes from commit messages into an output file Given an output file and set of revisions, this command will parse commit messages for release notes then add them to the output file. Release notes are defined in commit messages as ReStructuredText directives. These have the form:: .. directive:: title content Each ``directive`` maps to an output section in a generated release notes file, which itself is ReStructuredText. For example, the ``.. feature::`` directive would map to a ``New Features`` section. Release note directives can be either short-form or long-form. In short- form, ``title`` is omitted and the release note is rendered as a bullet list. In long form, a sub-section with the title ``title`` is added to the section. The ``FILE`` argument controls the output file to write gathered release notes to. The format of the file is:: Section 1 ========= ... Section 2 ========= ... Only sections with defined release notes are emitted. If a section only has short-form notes, it will consist of bullet list:: Section ======= * Release note 1 * Release note 2 If a section has long-form notes, sub-sections will be emitted:: Section ======= Note 1 Title ------------ Description of the first long-form note. Note 2 Title ------------ Description of the second long-form note. If the ``FILE`` argument points to an existing file, that file will be parsed for release notes having the format that would be generated by this command. The notes from the processed commit messages will be *merged* into this parsed set. During release notes merging: * Duplicate items are automatically ignored * Items that are different are automatically ignored if the similarity is greater than a threshold. This means that the release notes file can be updated independently from this command and changes should not be lost when running this command on that file. A particular use case for this is to tweak the wording of a release note after it has been added to the release notes file. """ sections = releasenotessections(ui) revs = scmutil.revrange(repo, [rev or 'not public()']) incoming = parsenotesfromrevisions(repo, sections.names(), revs) try: with open(file_, 'rb') as fh: notes = parsereleasenotesfile(sections, fh.read()) except IOError as e: if e.errno != errno.ENOENT: raise notes = parsedreleasenotes() notes.merge(ui, incoming) with open(file_, 'wb') as fh: fh.write(serializenotes(sections, notes)) @command('debugparsereleasenotes', norepo=True) def debugparsereleasenotes(ui, path): """parse release notes and print resulting data structure""" if path == '-': text = sys.stdin.read() else: with open(path, 'rb') as fh: text = fh.read() sections = releasenotessections(ui) notes = parsereleasenotesfile(sections, text) for section in notes: ui.write(_('section: %s\n') % section) for title, paragraphs in notes.titledforsection(section): ui.write(_(' subsection: %s\n') % title) for para in paragraphs: ui.write(_(' paragraph: %s\n') % ' '.join(para)) for paragraphs in notes.nontitledforsection(section): ui.write(_(' bullet point:\n')) for para in paragraphs: ui.write(_(' paragraph: %s\n') % ' '.join(para))