public LuaToken getToken()
{
int sequenceStart = -1;
int longBracketLength = -1;
bool isComment = false;
bool isHex = false;
bool isExp = false;
while (true)
{
switch (m_state)
{
case State.Start:
if (matchChar(EOF))
{
return(new LuaToken(LuaTokenType.EOF, new Byte[1], m_data.Length, getLine(m_data.Length)));
}
if (match("\""))
{
m_pos++;
sequenceStart = m_pos;
m_state = State.String;
continue;
}
if (match("--"))
{
m_pos += 2;
if (match("@"))
{
m_pos++;
continue;
}
m_state = State.Comment;
continue;
}
if (Char.IsDigit(peek()))
{
sequenceStart = m_pos;
m_state = State.Number;
isHex = false;
isExp = false;
continue;
}
if (Char.IsWhiteSpace(peek()))
{
m_pos++;
continue;
}
longBracketLength = getLongBracket();
if (longBracketLength != -1)
{
isComment = false;
m_state = State.LongBracketString;
sequenceStart = m_pos;
continue;
}
foreach (Tuple <string, LuaTokenType> op in Ops)
{
if (match(op.Item1))
{
LuaToken rst = new LuaToken(op.Item2, System.Text.Encoding.UTF8.GetBytes(op.Item1), m_pos, getLine(m_pos));
m_pos += op.Item1.Length;
return(rst);
}
}
if (matchChar(IdentifierStart))
{
m_state = State.Identifier;
sequenceStart = m_pos;
continue;
}
m_pos++;
continue;
case State.Identifier:
if (matchChar(IdentifierChars))
{
m_pos++;
continue;
}
Byte[] str = SubArray(m_data, sequenceStart, m_pos - sequenceStart);
foreach (Tuple <string, LuaTokenType> kw in Keywords)
{
if (equal(kw.Item1, str))
{
LuaToken rst = new LuaToken(kw.Item2, System.Text.Encoding.UTF8.GetBytes(kw.Item1), sequenceStart, getLine(sequenceStart));
m_state = State.Start;
return(rst);
}
}
m_state = State.Start;
return(new LuaToken(LuaTokenType.Identifier, str, sequenceStart, getLine(sequenceStart)));
case State.Comment:
{
longBracketLength = getLongBracket();
if (longBracketLength != -1)
{
isComment = true;
m_state = State.LongBracketString;
sequenceStart = m_pos;
continue;
}
while (!match("\n") && !match(EOF))
{
m_pos++;
}
if (match("\n"))
{
m_pos++;
}
m_state = State.Start;
continue;
}
case State.LongBracketString:
{
int l = -1;
while (!match(EOF))
{
l = matchLongBracket(longBracketLength);
if (l != -1)
{
break;
}
else
{
m_pos++;
}
}
Byte[] rst;
if (l == -1)
{
//EOF
rst = SubArray(m_data, sequenceStart);
}
else
{
rst = SubArray(m_data, sequenceStart, m_pos - sequenceStart);
m_pos += l;
}
m_state = State.Start;
if (isComment)
{
continue;
}
return(new LuaToken(LuaTokenType.StringLiteral, rst, sequenceStart, getLine(sequenceStart)));
}
case State.String:
while (!match(EOF) && !match("\""))
{
if (match("\\"))
{
m_pos++;
if (match("\""))
{
m_pos++;
}
}
else
{
m_pos++;
}
}
m_state = State.Start;
if (match(EOF))
{
return(new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart), sequenceStart, getLine(sequenceStart)));
}
else
{
m_pos++;
return(new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart, m_pos - sequenceStart - 1), sequenceStart, getLine(sequenceStart)));
}
case State.Number:
if (match("0x") || match("0X"))
{
m_pos += 2;
while (matchChar("0123456789abcdefABCDEF"))
{
m_pos++;
}
}
else
{
while (Char.IsDigit(peek()))
{
m_pos++;
}
if (match("."))
{
m_pos++;
while (Char.IsDigit(peek()))
{
m_pos++;
}
}
if (match("e") || match("E"))
{
m_pos++;
if (match("+") || match("-"))
{
m_pos++;
}
while (Char.IsDigit(peek()))
{
m_pos++;
}
}
}
m_state = State.Start;
return(new LuaToken(LuaTokenType.Number, SubArray(m_data, sequenceStart, m_pos - sequenceStart), sequenceStart, getLine(sequenceStart)));
default:
throw new Exception("unimplemented state!");
}
}
}
private void parseVariables()
{
{
LuaTokenizer lt = new LuaTokenizer(m_source.RawText, 0);
List <LuaToken> tokens = new List <LuaToken>();
while (true)
{
LuaToken t = lt.getToken();
tokens.Add(t);
if (t.Type == LuaTokenType.EOF)
{
break;
}
}
//System.Diagnostics.Debug.Print("=========");
LuaParser lp = new LuaParser(tokens);
LuaAST chunk = lp.parse();
//chunk.print(0);
LuaASTWalker walker = new LuaASTWalker();
if (lp.errMsg != null)
{
msg = lp.errMsg;
}
walker.walk(chunk, result);
}
/*
*
* int pos = 0;
* Byte[] str = m_source.RawText;
*
* for (; pos < str.Length; pos++)
* {
* char c = Convert.ToChar(str[pos]);
*
* //search for assignment operator
*
* if (!Parser.isCode(m_source, pos))
* {
* continue;
* }
*
* if (c != '=') continue;
* if (pos > 0)
* {
* if (Convert.ToChar(str[pos - 1]) == '=') continue;
* }
* if (pos < str.Length - 1)
* {
* if (Convert.ToChar(str[pos + 1]) == '=') continue;
* }
* MemberChain v = MemberChain.ParseBackward(m_source, pos - 1);
* if (v.Elements.Count > 1 || v.Elements.Count == 0) continue;
*
* string varName = v.getLastElement();
* Variable var = result.Variables.getVariable(varName);
* // if (var != null) continue;
*
* MemberChain elem = MemberChain.ParseFoward(m_source, pos + 1);
* if (elem == null) continue;
* Type t = elem.getType(result);
* if (t == null) continue;
* //if (t.displa == "") continue;
* //System.Diagnostics.Debug.Print(varName + " added");
*
* var = new Variable(varName);
* var.IsStatic = false;
* var.Type = t;
* var.StartPos = v.StartPos;
* var.EndPos = elem.EndPos;
* result.Variables.add(var);
* }*/
}
public LuaToken getToken() {
int sequenceStart = -1;
int longBracketLength = -1;
bool isComment = false;
bool isHex = false;
bool isExp = false;
while (true)
{
switch (m_state)
{
case State.Start:
if (matchChar(EOF)) {
return new LuaToken(LuaTokenType.EOF, new Byte[1], m_data.Length,getLine(m_data.Length));
}
if (match("\"")) {
m_pos++;
sequenceStart = m_pos;
m_state = State.String;
continue;
}
if(match("--")){
m_pos+=2;
if (match("@")) {
m_pos++;
continue;
}
m_state=State.Comment;
continue;
}
if (Char.IsDigit(peek())) {
sequenceStart = m_pos;
m_state = State.Number;
isHex = false;
isExp = false;
continue;
}
if(Char.IsWhiteSpace(peek())){
m_pos++;
continue;
}
longBracketLength = getLongBracket();
if (longBracketLength != -1) {
isComment = false;
m_state = State.LongBracketString;
sequenceStart = m_pos;
continue;
}
foreach (Tuple<string, LuaTokenType> op in Ops)
{
if (match(op.Item1))
{
LuaToken rst = new LuaToken(op.Item2, System.Text.Encoding.UTF8.GetBytes(op.Item1), m_pos, getLine(m_pos));
m_pos += op.Item1.Length;
return rst;
}
}
if (matchChar(IdentifierStart))
{
m_state = State.Identifier;
sequenceStart = m_pos;
continue;
}
m_pos++;
continue;
case State.Identifier:
if (matchChar(IdentifierChars)) {
m_pos++;
continue;
}
Byte[] str = SubArray(m_data,sequenceStart, m_pos - sequenceStart);
foreach (Tuple<string, LuaTokenType> kw in Keywords)
{
if (equal(kw.Item1,str))
{
LuaToken rst = new LuaToken(kw.Item2, System.Text.Encoding.UTF8.GetBytes(kw.Item1), sequenceStart, getLine(sequenceStart));
m_state = State.Start;
return rst;
}
}
m_state = State.Start;
return new LuaToken(LuaTokenType.Identifier, str, sequenceStart,getLine(sequenceStart));
case State.Comment:
{
longBracketLength = getLongBracket();
if (longBracketLength != -1)
{
isComment = true;
m_state = State.LongBracketString;
sequenceStart = m_pos;
continue;
}
while (!match("\n") && !match(EOF)) {
m_pos++;
}
if (match("\n")) m_pos++;
m_state = State.Start;
continue;
}
case State.LongBracketString:
{
int l = -1;
while (!match(EOF) ) {
l = matchLongBracket(longBracketLength);
if (l != -1)
{
break;
}
else
{
m_pos++;
}
}
Byte[] rst;
if (l == -1)
{
//EOF
rst = SubArray(m_data,sequenceStart);
}
else {
rst = SubArray(m_data,sequenceStart, m_pos - sequenceStart);
m_pos += l;
}
m_state = State.Start;
if (isComment) {
continue;
}
return new LuaToken(LuaTokenType.StringLiteral, rst, sequenceStart,getLine(sequenceStart));
}
case State.String:
while (!match(EOF) && !match("\"")) {
if (match("\\"))
{
m_pos++;
if (match("\""))
{
m_pos++;
}
}
else {
m_pos++;
}
}
m_state = State.Start;
if (match(EOF))
{
return new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data,sequenceStart), sequenceStart,getLine(sequenceStart));
}
else {
m_pos++;
return new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart, m_pos - sequenceStart - 1), sequenceStart, getLine(sequenceStart));
}
case State.Number:
if (match("0x") || match("0X"))
{
m_pos += 2;
while (matchChar("0123456789abcdefABCDEF"))
{
m_pos++;
}
}
else {
while (Char.IsDigit(peek())) {
m_pos++;
}
if (match(".")) {
m_pos++;
while (Char.IsDigit(peek()))
{
m_pos++;
}
}
if (match("e") || match("E")) {
m_pos++;
if (match("+") || match("-")) {
m_pos++;
}
while (Char.IsDigit(peek()))
{
m_pos++;
}
}
}
m_state = State.Start;
return new LuaToken(LuaTokenType.Number, SubArray(m_data, sequenceStart, m_pos - sequenceStart), sequenceStart, getLine(sequenceStart));
default:
throw new Exception("unimplemented state!");
}
}
}
