Geek asks and answers

Analysis of a tex-like language with lpeg

I'm struggling to bow my head to LPEG. I managed to create one grammar that does what I want, but I hit my head about it and did not go far. The idea is to analyze a document, which is a simplified form of TeX. I want to split the document into:

  • Environments that are pairs \begin{cmd}and \end{cmd}.
  • Teams who can take such an argument: \foo{bar}or may be naked: \foo.
  • Both the environment and the team can have the following parameters: \command[color=green,background=blue]{content}.
  • Other things.

I would also like to track line number information for error handling. Here is what I still have:

lpeg = require("lpeg")
lpeg.locale(lpeg)
-- Assume a lot of "X = lpeg.X" here.

-- Line number handling from http://lua-users.org/lists/lua-l/2011-05/msg00607.html
-- with additional print statements to check they are working.
local newline = P"\r"^-1 * "\n" / function (a) print("New"); end
local incrementline = Cg( Cb"linenum" )/ function ( a ) print("NL");  return a + 1 end , "linenum"
local setup = Cg ( Cc ( 1) , "linenum" )
nl = newline * incrementline
space = nl + lpeg.space

-- Taken from "Name-value lists" in http://www.inf.puc-rio.br/~roberto/lpeg/
local identifier = (R("AZ") + R("az") + P("_") + R("09"))^1
local sep = lpeg.S(",;") * space^0
local value = (1-lpeg.S(",;]"))^1
local pair = lpeg.Cg(C(identifier) * space ^0 * "=" * space ^0 * C(value)) * sep^-1
local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
local parameters = (P("[") * list * P("]")) ^-1

-- And the rest is mine

anything = C( (space^1 + (1-lpeg.S("\\{}")) )^1) * Cb("linenum") / function (a,b) return { text = a, line = b } end

begin_environment = P("\\begin") * Ct(parameters) * P("{") * Cg(identifier, "environment") * Cb("environment") * P("}") / function (a,b) return { params = a[1], environment = b } end
end_environment = P("\\end{") * Cg(identifier) * P("}") 

texlike = lpeg.P{
  "document";
  document = setup * V("stuff") * -1,
  stuff = Cg(V"environment" + anything + V"bracketed_stuff" + V"command_with" + V"command_without")^0,
  bracketed_stuff = P"{" * V"stuff" * P"}" / function (a) return a end,
  command_with =((P("\\") * Cg(identifier) * Ct(parameters) * Ct(V"bracketed_stuff"))-P("\\end{")) / function (i,p,n) return { command = i, parameters = p, nodes = n } end,
  command_without = (( P("\\") * Cg(identifier) * Ct(parameters) )-P("\\end{")) / function (i,p) return { command = i, parameters = p } end,
  environment = Cg(begin_environment * Ct(V("stuff")) * end_environment) / function (b,stuff, e) return { b = b, stuff = stuff, e = e} end
}

It almost works!

> texlike:match("\\foo[one=two]thing\\bar")
{
  command = "foo",
  parameters = {
    {
      one = "two",
    },
  },
}
{
  line = 1,
  text = "thing",
}
{
  command = "bar",
  parameters = {
  },
}

! -, . incrementline .

, ( Cg, C Ct ). , command_with :

> texlike:match("\\foo{text \\command moretext}")
{
  command = "foo",
  nodes = {
    {
      line = 1,
      text = "text ",
    },
  },
  parameters = {
  },
}

, , , "begin" , "". , .

+3
1

, , , , , .

, .

:

local incrementline = Cg( Cb"linenum" ) / 
                      function ( a ) print("NL");  return a + 1 end, 
                      "linenum"

, , . Backcapture linenum . , linenum - function(a) 1 . ) , "linenum" :

local incrementline = Cg( Cb"linenum" / 
                      function ( a ) print("NL");  return a + 1 end, 
                      "linenum")

LPeg Cg .

anything:

anything = C( (space^1 + (1-lpeg.S("\\{}")) )^1) * Cb("linenum") ...

. -, Cg ( incrementline ) , , . , adhoc, ​​ . , , - , :

C( (space^1 + (...) )^1)

, , * Cb("linenum"), - linenum, , .

, LPeg re , :

local grammar_cb =
{
  fold = pairfold, 
  resetlinenum = resetlinenum,
  incrementlinenum = incrementlinenum, getlinenum = getlinenum, 
  error = error
}

local texlike_grammar = re.compile(
[[
  document    <- '' -> resetlinenum {| docpiece* |} !.
  docpiece    <- {| envcmd |} / {| cmd |} / multiline
  beginslash  <- cmdslash 'begin'
  endslash    <- cmdslash 'end'
  envcmd      <- beginslash paramblock? {:beginenv: envblock :} (!endslash docpiece)*
                 endslash openbrace {:endenv: =beginenv :} closebrace / &beginslash {} -> error .
  envblock    <- openbrace key closebrace
  cmd         <- cmdslash {:command: identifier :} (paramblock? cmdblock)?
  cmdblock    <- openbrace {:nodes: {| docpiece* |} :} closebrace
  paramblock  <- opensq ( {:parameters: {| parampairs |} -> fold :} / whitesp) closesq
  parampairs  <- parampair (sep parampair)*
  parampair   <- key assign value
  key         <- whitesp { identifier }
  value       <- whitesp { [^],;%s]+ }
  multiline   <- (nl? text)+
  text        <- {| {:text: (!cmd !closebrace !%nl [_%w%p%s])+ :} {:line: '' -> getlinenum :} |}
  identifier  <- [_%w]+
  cmdslash    <- whitesp '\'
  assign      <- whitesp '='
  sep         <- whitesp ','
  openbrace   <- whitesp '{'
  closebrace  <- whitesp '}'
  opensq      <- whitesp '['
  closesq     <- whitesp ']'
  nl          <- {%nl+} -> incrementlinenum
  whitesp     <- (nl / %s)*
]], grammar_cb)

:

local function pairfold(...)
  local t, kv = {}, ...
  if #kv % 2 == 1 then return ... end
  for i = #kv, 2, -2 do
    t[ kv[i - 1] ] = kv[i]
  end
  return t
end

local incrementlinenum, getlinenum, resetlinenum do
  local line = 1
  function incrementlinenum(nl)
    assert(not nl:match "%S")
    line = line + #nl
  end

  function getlinenum() return line end
  function resetlinenum() line = 1 end
end

tex- str :

  local test1 = [[\foo{text \bar[color = red, background =   black]{
  moretext \baz{
even 
more text} }


this time skipping multiple

lines even, such wow!}]]

AST lua-table:

{
  command = "foo",
  nodes = {
    {
      text = "text",
      line = 1
    },
    {
      parameters = {
        color = "red",
        background = "black"
      },
      command = "bar",
      nodes = {
        {
          text = "  moretext",
          line = 2
        },
        {
          command = "baz",
          nodes = {
            {
              text = "even ",
              line = 3
            },
            {
              text = "more text",
              line = 4
            }
          }
        }
      }
    },
    {
      text = "this time skipping multiple",
      line = 7
    },
    {
      text = "lines even, such wow!",
      line = 9
    }
  }
}

/:

  local test2 = [[\begin[p1
=apple,
p2=blue]{scope} scope foobar   
\end{scope} global foobar]]

, , , :

{
  {
    {
      text = " scope foobar",
      line = 3
    },
    parameters = {
      p1 = "apple",
      p2 = "blue"
    },
    beginenv = "scope",
    endenv = "scope"
  },
  {
    text = " global foobar",
    line = 4
  }
}
+6

More articles:


All Articles