# HG changeset patch
# User Yuya Nishihara <yuya@tcha.org>
# Date 1434815366 -32400
# Node ID af329a84310c3d784b3717395b6fc4a32ca736f2
# Parent  9d1e04f5dca72f39233bc663fd31866c3610f3d3
parser: accept iterator of tokens instead of tokenizer function and program

This can simplify the interface of parse() function. Our tokenizer tends to
have optional arguments other than the message to be parsed.

Before this patch, the "lookup" argument existed only for the revset, and the
templater had to pack [program, start, end] to be passed to its tokenizer.

diff -r 9d1e04f5dca7 -r af329a84310c mercurial/fileset.py
--- a/mercurial/fileset.py	Sat Jun 20 20:11:53 2015 -0700
+++ b/mercurial/fileset.py	Sun Jun 21 00:49:26 2015 +0900
@@ -80,8 +80,8 @@
     yield ('end', None, pos)
 
 def parse(expr):
-    p = parser.parser(tokenize, elements)
-    tree, pos = p.parse(expr)
+    p = parser.parser(elements)
+    tree, pos = p.parse(tokenize(expr))
     if pos != len(expr):
         raise error.ParseError(_("invalid token"), pos)
     return tree
diff -r 9d1e04f5dca7 -r af329a84310c mercurial/parser.py
--- a/mercurial/parser.py	Sat Jun 20 20:11:53 2015 -0700
+++ b/mercurial/parser.py	Sun Jun 21 00:49:26 2015 +0900
@@ -19,8 +19,7 @@
 from i18n import _
 
 class parser(object):
-    def __init__(self, tokenizer, elements, methods=None):
-        self._tokenizer = tokenizer
+    def __init__(self, elements, methods=None):
         self._elements = elements
         self._methods = methods
         self.current = None
@@ -72,12 +71,9 @@
                     if len(infix) == 3:
                         self._match(infix[2], pos)
         return expr
-    def parse(self, message, lookup=None):
-        'generate a parse tree from a message'
-        if lookup:
-            self._iter = self._tokenizer(message, lookup)
-        else:
-            self._iter = self._tokenizer(message)
+    def parse(self, tokeniter):
+        'generate a parse tree from tokens'
+        self._iter = tokeniter
         self._advance()
         res = self._parse()
         token, value, pos = self.current
@@ -87,9 +83,9 @@
         if not isinstance(tree, tuple):
             return tree
         return self._methods[tree[0]](*[self.eval(t) for t in tree[1:]])
-    def __call__(self, message):
-        'parse a message into a parse tree and evaluate if methods given'
-        t = self.parse(message)
+    def __call__(self, tokeniter):
+        'parse tokens into a parse tree and evaluate if methods given'
+        t = self.parse(tokeniter)
         if self._methods:
             return self.eval(t)
         return t
diff -r 9d1e04f5dca7 -r af329a84310c mercurial/revset.py
--- a/mercurial/revset.py	Sat Jun 20 20:11:53 2015 -0700
+++ b/mercurial/revset.py	Sun Jun 21 00:49:26 2015 +0900
@@ -2387,9 +2387,9 @@
     >>> _parsealiasdecl('foo($1, $2, $1)')
     ('foo', None, None, 'argument names collide with each other')
     """
-    p = parser.parser(_tokenizealias, elements)
+    p = parser.parser(elements)
     try:
-        tree, pos = p.parse(decl)
+        tree, pos = p.parse(_tokenizealias(decl))
         if (pos != len(decl)):
             raise error.ParseError(_('invalid token'), pos)
 
@@ -2478,8 +2478,8 @@
                                            pos)
             yield (t, value, pos)
 
-    p = parser.parser(tokenizedefn, elements)
-    tree, pos = p.parse(defn)
+    p = parser.parser(elements)
+    tree, pos = p.parse(tokenizedefn(defn))
     if pos != len(defn):
         raise error.ParseError(_('invalid token'), pos)
     return parser.simplifyinfixops(tree, ('or',))
@@ -2609,8 +2609,8 @@
         return tuple(foldconcat(t) for t in tree)
 
 def parse(spec, lookup=None):
-    p = parser.parser(tokenize, elements)
-    tree, pos = p.parse(spec, lookup=lookup)
+    p = parser.parser(elements)
+    tree, pos = p.parse(tokenize(spec, lookup=lookup))
     if pos != len(spec):
         raise error.ParseError(_("invalid token"), pos)
     return parser.simplifyinfixops(tree, ('or',))
diff -r 9d1e04f5dca7 -r af329a84310c mercurial/templater.py
--- a/mercurial/templater.py	Sat Jun 20 20:11:53 2015 -0700
+++ b/mercurial/templater.py	Sun Jun 21 00:49:26 2015 +0900
@@ -27,8 +27,7 @@
     "end": (0, None, None),
 }
 
-def tokenizer(data):
-    program, start, end = data
+def tokenize(program, start, end):
     pos = start
     while pos < end:
         c = program[pos]
@@ -96,7 +95,7 @@
 def compiletemplate(tmpl, context):
     parsed = []
     pos, stop = 0, len(tmpl)
-    p = parser.parser(tokenizer, elements)
+    p = parser.parser(elements)
     while pos < stop:
         n = tmpl.find('{', pos)
         if n < 0:
@@ -111,8 +110,7 @@
         if n > pos:
             parsed.append(('string', tmpl[pos:n]))
 
-        pd = [tmpl, n + 1, stop]
-        parseres, pos = p.parse(pd)
+        parseres, pos = p.parse(tokenize(tmpl, n + 1, stop))
         parsed.append(parseres)
 
     return [compileexp(e, context, methods) for e in parsed]