Commit: f0bc39d34f56083b4ca3345023730d8ded559f50 - CCastle2 (hg)

Castle: The best Real-Time/Embedded/HighTech language EVER. Attempt 2

Commit MetaInfo

• Révision: f0bc39d34f56083b4ca3345023730d8ded559f50 (tree)
• l'heure: 2022-01-03 05:43:48
• Auteur: Albert Mietus < albert AT mietus DOT nl >
• Commiter: Albert Mietus < albert AT mietus DOT nl >

Message de Log

Added Quantification (former: 'many') suffix as '*','+','?'. --Needed to move the optional part to a sub-rule, to be able to visit it

Change Summary

• modified: AST/castle/peg.py
• modified: AST/pytst/test_0.py
• modified: Arpeggio/demo/test_Rule.py
• modified: Arpeggio/grammar.py
• modified: Arpeggio/pytst/d1_parse_tree/test_1_regexp.py
• modified: Arpeggio/pytst/d1_parse_tree/test_2_str.py
• modified: Arpeggio/pytst/d1_parse_tree/test_3_expressions.py
• modified: Arpeggio/pytst/d1_parse_tree/test_4_rule.py
• modified: Arpeggio/pytst/d2_ast/__init__.py
• modified: Arpeggio/pytst/d2_ast/test_1_term.py
• modified: Arpeggio/pytst/d2_ast/test_2_ID.py
• modified: Arpeggio/pytst/d2_ast/test_3_rule.py
• modified: Arpeggio/visitor.py
• add: Arpeggio/pytst/d2_ast/test_4_quantity.py

Modification

diff -r 7dbab792d37e -r f0bc39d34f56 AST/castle/peg.py
--- a/AST/castle/peg.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/AST/castle/peg.py	Sun Jan 02 21:43:48 2022 +0100
@@ -61,12 +61,19 @@
         self.settings = settings
 
 
+class Group(Expression):pass                                            # abstract
+class UnorderedGroup(Group):pass                                        # It looks like a Quantity, but is a group
 
-class ManyExpression(Expression): pass                                  # abstract
-class Group(Expression):pass
+
+class Quantity(Expression):                                             # abstract
+    """An expression with Quantification; like optional, or repetition. The subclasses defines which Quantification"""
+    def __init__(self, *, expr=None, **kwargs):
+        super().__init__(**kwargs)
+        self.expr = expr
+
 
 class Sequence(Expression):
-    """A "list of expressions; can be of length=1"""
+    """A _list_ of expressions; can be of length=1"""
     def __init__(self, *, value=None, **kwargs):
         super().__init__(**kwargs)
         self.value=value
@@ -76,12 +83,14 @@
         return self.value[n]
 
 
-class OrderedChoice(Expression):pass
-class Predicate(Expression): pass                                       # abstract
+class OrderedChoice(Expression):pass                                    # It a an set of alternatives
 
-class Optional(ManyExpression):pass
-class OneOrMore(ManyExpression):pass
-class ZeroOrMore(ManyExpression):pass
 
+class Optional(Quantity):pass
+class ZeroOrMore(Quantity):pass
+class OneOrMore(Quantity):pass
+
+
+class Predicate(Expression): pass                                       # abstract
 class AndPredicate(Predicate): pass
 class NotPredicate(Predicate): pass

diff -r 7dbab792d37e -r f0bc39d34f56 AST/pytst/test_0.py
--- a/AST/pytst/test_0.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/AST/pytst/test_0.py	Sun Jan 02 21:43:48 2022 +0100
@@ -10,7 +10,7 @@
 
 def test_abstracts_2():
     assert isinstance(peg.Expression(), peg.NonTerminal)
-    assert isinstance(peg.ManyExpression(), peg.Expression)
+    assert isinstance(peg.Quantity(), peg.Expression)
     assert isinstance(peg.Predicate(), peg.Expression)
 
 def test_eof():
@@ -21,6 +21,6 @@
     assert isinstance(peg.NotPredicate(), peg.Predicate)
 
 def test_manys():
-    assert isinstance(peg.Optional(), peg.ManyExpression)
-    assert isinstance(peg.OneOrMore(), peg.ManyExpression)
-    assert isinstance(peg.ZeroOrMore(), peg.ManyExpression)
+    assert isinstance(peg.Optional(), peg.Quantity)
+    assert isinstance(peg.OneOrMore(), peg.Quantity)
+    assert isinstance(peg.ZeroOrMore(), peg.Quantity)

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/demo/test_Rule.py
--- a/Arpeggio/demo/test_Rule.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/demo/test_Rule.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,4 +1,5 @@
 import pytest
+import logging;logger = logging.getLogger(__name__)
 
 import grammar
 import arpeggio
@@ -13,7 +14,7 @@
 
 def show_visited(label="VISIT_", node=None, children=None):
     nl= "\n" if label[-1] != '_' else ""
-    print(f'XXX {label}{nl}{format_node(node)}{format_children(children)}')
+    logger.info(f'{label}{nl}{format_node(node)}{format_children(children)}')
 
 
 

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/grammar.py
--- a/Arpeggio/grammar.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/grammar.py	Sun Jan 02 21:43:48 2022 +0100
@@ -7,8 +7,9 @@
 def rule():		return rule_name, '<-', expressions, ";"
 
 def expressions():	return ( OneOrMore(single_expr), Optional( '|' , expressions ) )
-def single_expr():	return ( [ rule_crossref, term, group, predicate ], Optional([ '?' , '*' , '+' , '#' ]))
+def single_expr():	return ( [ rule_crossref, term, group, predicate ], expr_quantity)
 
+def expr_quantity(): 	return Optional([ '?' , '*' , '+' , '#' ])
 def term():		return [ str_term, regex_term ]
 def group():		return '(', expressions, ')'
 def predicate():	return ['&','!'], single_expr

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d1_parse_tree/test_1_regexp.py
--- a/Arpeggio/pytst/d1_parse_tree/test_1_regexp.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d1_parse_tree/test_1_regexp.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,11 +1,13 @@
 import pytest
+import logging;logger = logging.getLogger(__name__)
+
 from grammar import *
 
 import arpeggio
 RE, S = arpeggio.RegExMatch, arpeggio.StrMatch                          # shortcut
 
 def parse_regex(txt, pattern=None):
-    #print(f'\nXXX >>{txt}<<')
+    logger.debug(f'>>{txt}<<')
     parser = ParserPython(regex_term, comment) # DoNot use debug/dot_exporter as it will re-use-same-file
     tree = parser.parse(txt)
     assert tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[tree.position: tree.position_end]}<<; Not: >>{txt[tree.position_end:]}<<."

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d1_parse_tree/test_2_str.py
--- a/Arpeggio/pytst/d1_parse_tree/test_2_str.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d1_parse_tree/test_2_str.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,4 +1,5 @@
 import pytest
+import logging;logger = logging.getLogger(__name__)
 
 from grammar import *
 import arpeggio
@@ -6,7 +7,7 @@
 
 
 def parse_str(str, pattern=[S, RE, S]):
-    #print(f'\nXXX >>{str}<<')
+    logger.debug(f'>>{str}<<')
     parser = ParserPython(str_term, comment)
     tree = parser.parse(str)
     assert tree.position_end == len(str) , f"Not parsed whole input; Only: >>{str[tree.position: tree.position_end]}<<; Not: >>{str[tree.position_end:]}<<."

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d1_parse_tree/test_3_expressions.py
--- a/Arpeggio/pytst/d1_parse_tree/test_3_expressions.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d1_parse_tree/test_3_expressions.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,6 +1,7 @@
 import pytest
+import logging;logger = logging.getLogger(__name__)
+
 import grammar
-
 import arpeggio
 
 R, S, X = grammar.regex_term.__name__, grammar.str_term.__name__, grammar.rule_crossref.__name__  # shortcut in grammar
@@ -10,7 +11,7 @@
 def parse_expressions(txt, pattern=None):
     parser = arpeggio.ParserPython(grammar.expressions)
     parse_tree = parser.parse(txt)
-    print("\nPARSE-TREE\n" + parse_tree.tree_str()+'\n')
+    logger.info("\nPARSE-TREE\n" + parse_tree.tree_str()+'\n')
 
     assert parse_tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[parse_tree.position: parse_tree.position_end]}<<; Not: >>{txt[parse_tree.position_end:]}<<."
     assert parse_tree.rule_name == "expressions"
@@ -20,7 +21,6 @@
     return parse_tree
 
 def validate_pattern(pt, pattern=None):
-    print('VVV', pt, type(pt))
     assert len(pt) == len(pattern), f"Not correct number-of-element"
 
     for p, s in zip(pattern, pt): # E <- S* (| E)?
@@ -31,14 +31,11 @@
             assert s[0][0].rule_name == p  # S => T => str/regex
         elif isinstance(p, tuple):         # Group: '(' ... ')'
             assert s[0].rule_name == G
-            print("\nVVV:G\n" + s.tree_str())
-            # XXXXX
             validate_pattern(s[0][1:-1][0], pattern=p) # G=>E=>
         elif p == P:
             assert False, "To Do: Predicate"
         else:
             assert False, "To Do: More"
-    print('VVV OKE')
 
 
 def test_simple_1():	parse_expressions(r"abc",	pattern=[X])

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d1_parse_tree/test_4_rule.py
--- a/Arpeggio/pytst/d1_parse_tree/test_4_rule.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d1_parse_tree/test_4_rule.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,4 +1,6 @@
 import pytest
+import logging;logger = logging.getLogger(__name__)
+
 import grammar
 
 import arpeggio
@@ -7,7 +9,7 @@
 def parse_rule(txt, pattern=None):
     parser = arpeggio.ParserPython(grammar.rule)
     tree = parser.parse(txt)
-    print(f'\nTREE\n{tree.tree_str()}')
+    logger.info(f'\nTREE\n{tree.tree_str()}')
 
     assert tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[tree.position: tree.position_end]}<<; Not: >>{txt[tree.position_end:]}<<."
     assert len(tree) == 4, 		      		"A rule should have length=4; ..."

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d2_ast/__init__.py
--- a/Arpeggio/pytst/d2_ast/__init__.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d2_ast/__init__.py	Sun Jan 02 21:43:48 2022 +0100
@@ -4,11 +4,15 @@
 import sys; sys.path.append("./../AST/") ; sys.path.append("./../../AST/")
 from castle import peg # has the AST clases
 
-def parse(txt, rule):
+def parse(txt, rule,
+          print_tree_debug=False,
+          visitor_debug=False):
     parser = arpeggio.ParserPython(rule)
     pt = parser.parse(txt)
+    if print_tree_debug:
+        print('\n'+pt.tree_str())
     assert pt.position_end == len(txt), f"Did not parse all input txt=>>{txt}<<len={len(txt)} ==> parse_tree: >>{pt}<<_end={pt.position_end}"
-    ast = arpeggio.visit_parse_tree(pt, visitor.PegVisitor())
+    ast = arpeggio.visit_parse_tree(pt, visitor.PegVisitor(debug=visitor_debug))
     assert ast.position == 0 and ast.position_end == len(txt), f"Also the AST (type={type(ast)}) should include all input"
     return ast
 
@@ -18,3 +22,5 @@
     assert isinstance(id, peg.ID),		"The id should be an ID"
     peg.ID.validate_or_raise(id)                # with correct syntax
     assert id.name == name, err_message if err_message  else f"Note correct name, expected {name}"
+
+

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d2_ast/test_1_term.py
--- a/Arpeggio/pytst/d2_ast/test_1_term.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d2_ast/test_1_term.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,7 +1,7 @@
 import pytest
 
 import grammar
-from castle import peg # has the AST clases
+from castle import peg # has the AST classes
 
 from . import parse
 

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d2_ast/test_2_ID.py
--- a/Arpeggio/pytst/d2_ast/test_2_ID.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d2_ast/test_2_ID.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,7 +1,7 @@
 import pytest
 
 import grammar
-from castle import peg # has the AST clases
+from castle import peg # has the AST classes
 
 from . import parse, assert_ID
 

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d2_ast/test_3_rule.py
--- a/Arpeggio/pytst/d2_ast/test_3_rule.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/pytst/d2_ast/test_3_rule.py	Sun Jan 02 21:43:48 2022 +0100
@@ -1,7 +1,7 @@
 import pytest
 
 import grammar
-from castle import peg # has the AST clases
+from castle import peg # has the AST classes
 
 from . import parse, assert_ID
 

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/pytst/d2_ast/test_4_quantity.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Arpeggio/pytst/d2_ast/test_4_quantity.py	Sun Jan 02 21:43:48 2022 +0100
@@ -0,0 +1,35 @@
+"""  Test several optional parts of an expression -- mosty quantity suffixes like '?' '*' and '+' -- also '#' although is different"""
+
+import pytest
+import logging;logger = logging.getLogger(__name__)
+
+import grammar
+from castle import peg # has the AST classes
+
+from . import parse, assert_ID
+
+def assert_Quantification(token:str, kind:type(peg.Quantity)):
+    txt = f"R <- X {token} ;"
+
+    ast = parse(txt, grammar.rule)
+    assert_ID(ast.name, 'R')
+
+    expr = ast.expr
+    assert len(expr)==1,			"There should be only one expr"
+
+    assert isinstance(expr[0], peg.Quantity), 	"It should be a (sub of) Quantity .."
+    assert isinstance(expr[0], kind),		f"... namely the specified one: {kind}"
+
+    opt_ex = expr[0].expr
+    assert_ID(opt_ex, 'X', "should be same a the expr; without the Quantification")
+
+
+def test_Optional():
+    assert_Quantification('?', peg.Optional)
+
+def test_ZeroOrMore():
+    assert_Quantification('*', peg.ZeroOrMore)
+
+def test_OneOrMore():
+    assert_Quantification('+', peg.OneOrMore)
+

diff -r 7dbab792d37e -r f0bc39d34f56 Arpeggio/visitor.py
--- a/Arpeggio/visitor.py	Sat Jan 01 20:38:16 2022 +0100
+++ b/Arpeggio/visitor.py	Sun Jan 02 21:43:48 2022 +0100
@@ -3,7 +3,15 @@
 import sys; sys.path.append("./../AST/") ; sys.path.append("./../../AST/")
 from castle import peg # has the AST clases
 
+import logging;logger = logging.getLogger(__name__)
+
+class QuantityError(ValueError): pass
+
 class PegVisitor(arpeggio.PTNodeVisitor):
+    token_2_class = {'?': peg.Optional,
+                     '*': peg.ZeroOrMore,
+                     '+': peg.OneOrMore,
+                     '#': peg.UnorderedGroup}
 
     def visit_str_term(self, node, children):
         return peg.StrTerm(value=node[1], parse_tree=node)
@@ -20,15 +28,34 @@
     def visit_rule(self, node, children):                               #  Name '<-' expressions ';'
         return peg.Rule(name=children[0],expr=children[1], parse_tree=node)
 
+    def visit_single_expr(self, node, children):                        #  [ rule_crossref, term, group, predicate ],  expr_quantity
+        if len(children) == 1: #  No Optional part
+            try:
+                n = children[0].name
+            except AttributeError:
+                n = str(children[0])
+            logger.warning(f'visit_single_expr==1:: {n}:{type(children[0])}')
+            return children[0]
+        elif len(children) == 2: #  Optional part
+            logger.debug(f'visit_single_expr==2:: {children[0]}, {children[1]}')
+            expr = children[0]
+            token = str(children[1])
+            quantum_cls = self.token_2_class.get(token)
+            if quantum_cls:
+                ast=quantum_cls(expr=expr, parse_tree=node)
+                logger.debug(f'visit_single_expr==2:: {quantum_cls} {expr}')
+                return ast
+            else:
+                raise QuantityError(f"token '{token}' not recognised")
+        else:
+            raise  NotImplementedError("visit_single_expr, len>2")      # XXX -- Is this possible?
 
-#    def visit_single_expr(self, node, children):                        #  [ rule_crossref, term, group, predicate ] Optional([ '?' , '*' , '+' , '#' ]))
-#        if len(children) == 1: # No optional part
-#            #ast = peg.Sequence(value=children, parse_tree=node)
-#            return super().visit__default__(node, children)
-#        else:
-#            assert NotImplementedError("To Do: visit_single_expr with optional part")  # XXX
-#        return ast
+    def visit_expr_quantity(self, node, children):                      # Optional([ '?' , '*' , '+' , '#' ])
+        logger.debug(f'visit_expr_quantity [{len(children)}] node={node}:{type(node)}')
+        return node
 
-    def visit_expressions(self, node, children):                        #   OneOrMore(single_expr), Optional( '|' , expressions )
+
+    def visit_expressions(self, node, children):                        # OneOrMore(single_expr), Optional( '|' , expressions )
+        logger.debug(f'>>{node}<< len={len(children)} children={children}')
         return peg.Sequence(value=children, parse_tree=node)