Geek asks and answers

ANTLR get and split lexer content

first, sorry for my english, I'm still learning.

I am writing a Python module for my framework that parses CSS files. I am trying regex, ply (python lexer and parser), but I ended up in ANTLR.

Try it first, I need to parse the comments from the CSS file. This is my CSS parsing line:

/*test*/

/*
test1
/*

/*test2/*nested*/comment/*

I know that CSS does not allow nested comments, but I need it within my framework. I wrote a simple ANTLR grammar:

grammar CSS;

options {
    language = Python;
}

styleSheet
    : comments EOF ;

comments
    : NESTED_ML_COMMENT*
    ;

NESTED_ML_COMMENT
    :   '/*' 
        (options {greedy=false;} : (NESTED_ML_COMMENT | . ) )* 
        '*/' 
    ;

LINEBREAK 
    :  ('\n\r' | '\n')+{$channel=HIDDEN; };

As a result, I get:

enter image description here

What I expect (work with paint: D):

enter image description here

Please note that I do not want / * and * / as a result.

Is there a way to do this in a pure ANTLR? I have no problem using python in ANTLR, but if there is a way to do it without python, I will be grateful.

+1
2

, . NESTED_ML_COMMENT ( "" ), , /*test2/*nested*/comment*/: lexer "" . , () (.. /* */), , .

, 2 nd , "" ( ). COMMENT_START : '/*'; COMMENT_END : '*/';. : , /* */.

, () CSS. CSS , -, -.

. :

grammar T;

parse
  :  comment EOF 
  ;

comment
  :  COMMENT_START (ANY | comment)* COMMENT_END
  ;

COMMENT_START : '/*';
COMMENT_END   : '*/';
ANY           :  . ;

/*test2/*nested*/comment*/ :

enter image description here

, /* */ , .

CSS :

comment
  :  NESTED_ML_COMMENT 
     {
       text = $NESTED_ML_COMMENT.text
       # invoke the TParser (my demo grammar) on `text`
     }
  ;

, ANTLRWorks , . ANTLR , ( ANTLRWorks , - ).

Q & A, , AST: AST, ANTLR?

"", . ANY TEXT. . /* */. , Python lexer, . TEXT , * , /, a / , a *:

grammar Comment;

options {
  output=AST;
  language=Python;
}

tokens {
  COMMENT;
}

@lexer::members {
  def not_part_of_comment(self):
    current = self.input.LA(1)
    next = self.input.LA(2)
    if current == ord('*'): return next != ord('/')
    if current == ord('/'): return next != ord('*')  
    return True
}

parse
  :  comment EOF -> comment
  ;

comment
  :  COMMENT_START atom* COMMENT_END -> ^(COMMENT atom*)
  ;

atom
  :  TEXT
  |  comment
  ;

COMMENT_START : '/*';
COMMENT_END   : '*/';
TEXT          : ({self.not_part_of_comment()}?=> . )+ ;

, { boolean_expression }?=>, Q & A: " " ANTLR?

, , Python (. ANTLR Wiki). ANTLR 3.1.3 .

:

java -cp antlr-3.1.3.jar org.antlr.Tool Comment.g

lexer parser Python script:

#!/usr/bin/env python

import antlr3
from antlr3 import *
from antlr3.tree import *
from CommentLexer import *
from CommentParser import *

# http://www.antlr.org/wiki/display/ANTLR3/Python+runtime
# http://www.antlr.org/download/antlr-3.1.3.jar

def print_level_order(tree, indent):
  print '{0}{1}'.format('   '*indent, tree.text)
  for child in tree.getChildren():
    print_level_order(child, indent+1)

input = '/*aaa1/*bbb/*ccc*/*/aaa2*/'
char_stream = antlr3.ANTLRStringStream(input)
lexer = CommentLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = CommentParser(tokens)
tree = parser.parse().tree 
print_level_order(tree, 0)

, "/*aaa1/*bbb/*ccc*/*/aaa2*/" AST:

COMMENT
   aaa1
   COMMENT
      bbb
      COMMENT
         ccc
   aaa2

EDIT II

, Comment CSS. :

grammar CSS;

options {
  output=AST;
  language=Python;
}

tokens {
  CSS_FILE;
  RULE;
  BLOCK;
  DECLARATION;
}

@parser::header {
import antlr3
from antlr3 import *
from antlr3.tree import *
from CommentLexer import *
from CommentParser import *
}

@parser::members {
  def parse_comment(self, text):
    lexer = CommentLexer(antlr3.ANTLRStringStream(text))
    parser = CommentParser(antlr3.CommonTokenStream(lexer))
    return parser.parse().tree 
}

parse
  :  atom+ EOF -> ^(CSS_FILE atom+)
  ;

atom
  :  rule
  |  Comment -> {self.parse_comment($Comment.text)}
  ;

rule
  :  Identifier declarationBlock -> ^(RULE Identifier declarationBlock)
  ;

declarationBlock
  :  '{' declaration+ '}' -> ^(BLOCK declaration+)
  ;

declaration
  :  a=Identifier ':' b=Identifier ';' -> ^(DECLARATION $a $b)
  ;

Identifier
  :  ('a'..'z' | 'A'..'Z') ('a'..'z' | 'A'..'Z' | '0'..'9')*
  ;

Comment
  :  '/*' (options {greedy=false;} : Comment | . )* '*/'
  ;

Space
  :  (' ' | '\t' | '\r' | '\n') {$channel=HIDDEN;}
  ;

:

h1 {  a: b;  c: d;}

/*aaa1/*bbb/*ccc*/*/aaa2*/

p {x  :  y;}

CSSParser :

CSS_FILE
   RULE
      h1
      BLOCK
         DECLARATION
            a
            b
         DECLARATION
            c
            d
   COMMENT
      aaa1
      COMMENT
         bbb
         COMMENT
            ccc
      aaa2
   RULE
      p
      BLOCK
         DECLARATION
            x
            y

, script:

#!/usr/bin/env python

import antlr3
from antlr3 import *
from antlr3.tree import *
from CSSLexer import *
from CSSParser import *

def print_level_order(tree, indent):
  print '{0}{1}'.format('   '*indent, tree.text)
  for child in tree.getChildren():
    print_level_order(child, indent+1)

input = 'h1 {  a: b;  c: d;}\n\n/*aaa1/*bbb/*ccc*/*/aaa2*/\n\np {x  :  y;}'
char_stream = antlr3.ANTLRStringStream(input)
lexer = CSSLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = CSSParser(tokens)
tree = parser.parse().tree 
print_level_order(tree, 0)
+2

! ^ AST. /* AST, ! . , AST, ^. :

NESTED_ML_COMMENT
:   COMMENT_START!
    (options {greedy=false;} : (NESTED_ML_COMMENT^ | . ) )* 
    COMMENT_END!
;

, , , , , : ^ ! ANTLR

0

More articles:


All Articles