Geek asks and answers

Boost spirit lex writes the value of the token back to the input stream

I am wondering if there is a way in boost :: spirit :: lex to write the token value back to the input stream (possibly after editing) and re-scan. I am mainly looking for functionality similar to the unput () function in Flex.

Thank!

+3
source share
2 answers

I completed the implementation of my own unput () function as follows:

   struct unputImpl
   {
      template <typename Iter1T, typename Iter2T, typename StrT>
      struct result {
         typedef void type;
      };

      template <typename Iter1T, typename Iter2T, typename StrT>
      typename result<Iter1T, Iter2T, StrT>::type operator()(Iter1T& start, Iter2T& end, StrT str) const {
         start -= (str.length() - std::distance(start, end));
         std::copy(str.begin(), str.end(), start);
         end = start;
      }
   };

   phoenix::function<unputImpl> const unput = unputImpl();

Then it can be used as:

   this->self += lex::token_def<lex::omit>("{SYMBOL}\\(")
        [
           unput(_start, _end, "(" + construct<string>(_start, _end - 1) + " "),
           _pass = lex::pass_flags::pass_ignore
        ];

, , . , , , , , , unput() .

0

, , .

, , , , . :

Input 'abc(' Parsed as: '(abc'
Input '(abc' Parsed as: '(abc'


#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <iostream>
#include <string>

using namespace boost::spirit;

///// LEXER
template <typename Lexer>
struct tokens : lex::lexer<Lexer>
{
    tokens()
    {
        identifier = "[a-zA-Z][a-zA-Z0-9]*";
        paren_open = '(';

        this->self.add
            (identifier)
            (paren_open)
            ;
    }

    lex::token_def<std::string> identifier;
    lex::token_def<lex::omit> paren_open;
};

///// GRAMMAR
template <typename Iterator>
struct grammar : qi::grammar<Iterator, std::string()>
{
    template <typename TokenDef>
        grammar(TokenDef const& tok) : grammar::base_type(ident_w_parenopen)
    {
        ident_w_parenopen = 
              (tok.identifier >> tok.paren_open)
            | (tok.paren_open >> tok.identifier) 
            ;
    }
  private:
    qi::rule<Iterator, std::string()> ident_w_parenopen;
};

///// DEMONSTRATION
typedef std::string::const_iterator It;

template <typename T, typename G>
void DoTest(std::string const& input, T const& tokens, G const& g)
{
    It first(input.begin()), last(input.end());

    std::string parsed;
    bool r = lex::tokenize_and_parse(first, last, tokens, g, parsed);

    if (r) {
        std::cout << "Input '" << input << "' Parsed as: '(" << parsed << "'\n";
    }
    else {
        std::string rest(first, last);
        std::cerr << "Parsing '" << input << "' failed\n" << "stopped at: \"" << rest << "\"\n";
    }
}

int main(int argc, char* argv[])
{
    typedef lex::lexertl::token<It, boost::mpl::vector<std::string> > token_type;
    typedef lex::lexertl::lexer<token_type> lexer_type;
    typedef tokens<lexer_type>::iterator_type iterator_type;

    tokens<lexer_type> tokens;
    grammar<iterator_type> g (tokens);

    DoTest("abc(", tokens, g);
    DoTest("(abc", tokens, g);
}
+3

More articles:


All Articles