public LuaToken getToken() { int sequenceStart = -1; int longBracketLength = -1; bool isComment = false; bool isHex = false; bool isExp = false; while (true) { switch (m_state) { case State.Start: if (matchChar(EOF)) { return(new LuaToken(LuaTokenType.EOF, new Byte[1], m_data.Length, getLine(m_data.Length))); } if (match("\"")) { m_pos++; sequenceStart = m_pos; m_state = State.String; continue; } if (match("--")) { m_pos += 2; if (match("@")) { m_pos++; continue; } m_state = State.Comment; continue; } if (Char.IsDigit(peek())) { sequenceStart = m_pos; m_state = State.Number; isHex = false; isExp = false; continue; } if (Char.IsWhiteSpace(peek())) { m_pos++; continue; } longBracketLength = getLongBracket(); if (longBracketLength != -1) { isComment = false; m_state = State.LongBracketString; sequenceStart = m_pos; continue; } foreach (Tuple <string, LuaTokenType> op in Ops) { if (match(op.Item1)) { LuaToken rst = new LuaToken(op.Item2, System.Text.Encoding.UTF8.GetBytes(op.Item1), m_pos, getLine(m_pos)); m_pos += op.Item1.Length; return(rst); } } if (matchChar(IdentifierStart)) { m_state = State.Identifier; sequenceStart = m_pos; continue; } m_pos++; continue; case State.Identifier: if (matchChar(IdentifierChars)) { m_pos++; continue; } Byte[] str = SubArray(m_data, sequenceStart, m_pos - sequenceStart); foreach (Tuple <string, LuaTokenType> kw in Keywords) { if (equal(kw.Item1, str)) { LuaToken rst = new LuaToken(kw.Item2, System.Text.Encoding.UTF8.GetBytes(kw.Item1), sequenceStart, getLine(sequenceStart)); m_state = State.Start; return(rst); } } m_state = State.Start; return(new LuaToken(LuaTokenType.Identifier, str, sequenceStart, getLine(sequenceStart))); case State.Comment: { longBracketLength = getLongBracket(); if (longBracketLength != -1) { isComment = true; m_state = State.LongBracketString; sequenceStart = m_pos; continue; } while (!match("\n") && !match(EOF)) { m_pos++; } if (match("\n")) { m_pos++; } m_state = State.Start; continue; } case State.LongBracketString: { int l = -1; while (!match(EOF)) { l = matchLongBracket(longBracketLength); if (l != -1) { break; } else { m_pos++; } } Byte[] rst; if (l == -1) { //EOF rst = SubArray(m_data, sequenceStart); } else { rst = SubArray(m_data, sequenceStart, m_pos - sequenceStart); m_pos += l; } m_state = State.Start; if (isComment) { continue; } return(new LuaToken(LuaTokenType.StringLiteral, rst, sequenceStart, getLine(sequenceStart))); } case State.String: while (!match(EOF) && !match("\"")) { if (match("\\")) { m_pos++; if (match("\"")) { m_pos++; } } else { m_pos++; } } m_state = State.Start; if (match(EOF)) { return(new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart), sequenceStart, getLine(sequenceStart))); } else { m_pos++; return(new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart, m_pos - sequenceStart - 1), sequenceStart, getLine(sequenceStart))); } case State.Number: if (match("0x") || match("0X")) { m_pos += 2; while (matchChar("0123456789abcdefABCDEF")) { m_pos++; } } else { while (Char.IsDigit(peek())) { m_pos++; } if (match(".")) { m_pos++; while (Char.IsDigit(peek())) { m_pos++; } } if (match("e") || match("E")) { m_pos++; if (match("+") || match("-")) { m_pos++; } while (Char.IsDigit(peek())) { m_pos++; } } } m_state = State.Start; return(new LuaToken(LuaTokenType.Number, SubArray(m_data, sequenceStart, m_pos - sequenceStart), sequenceStart, getLine(sequenceStart))); default: throw new Exception("unimplemented state!"); } } }
private void parseVariables() { { LuaTokenizer lt = new LuaTokenizer(m_source.RawText, 0); List <LuaToken> tokens = new List <LuaToken>(); while (true) { LuaToken t = lt.getToken(); tokens.Add(t); if (t.Type == LuaTokenType.EOF) { break; } } //System.Diagnostics.Debug.Print("========="); LuaParser lp = new LuaParser(tokens); LuaAST chunk = lp.parse(); //chunk.print(0); LuaASTWalker walker = new LuaASTWalker(); if (lp.errMsg != null) { msg = lp.errMsg; } walker.walk(chunk, result); } /* * * int pos = 0; * Byte[] str = m_source.RawText; * * for (; pos < str.Length; pos++) * { * char c = Convert.ToChar(str[pos]); * * //search for assignment operator * * if (!Parser.isCode(m_source, pos)) * { * continue; * } * * if (c != '=') continue; * if (pos > 0) * { * if (Convert.ToChar(str[pos - 1]) == '=') continue; * } * if (pos < str.Length - 1) * { * if (Convert.ToChar(str[pos + 1]) == '=') continue; * } * MemberChain v = MemberChain.ParseBackward(m_source, pos - 1); * if (v.Elements.Count > 1 || v.Elements.Count == 0) continue; * * string varName = v.getLastElement(); * Variable var = result.Variables.getVariable(varName); * // if (var != null) continue; * * MemberChain elem = MemberChain.ParseFoward(m_source, pos + 1); * if (elem == null) continue; * Type t = elem.getType(result); * if (t == null) continue; * //if (t.displa == "") continue; * //System.Diagnostics.Debug.Print(varName + " added"); * * var = new Variable(varName); * var.IsStatic = false; * var.Type = t; * var.StartPos = v.StartPos; * var.EndPos = elem.EndPos; * result.Variables.add(var); * }*/ }
public LuaToken getToken() { int sequenceStart = -1; int longBracketLength = -1; bool isComment = false; bool isHex = false; bool isExp = false; while (true) { switch (m_state) { case State.Start: if (matchChar(EOF)) { return new LuaToken(LuaTokenType.EOF, new Byte[1], m_data.Length,getLine(m_data.Length)); } if (match("\"")) { m_pos++; sequenceStart = m_pos; m_state = State.String; continue; } if(match("--")){ m_pos+=2; if (match("@")) { m_pos++; continue; } m_state=State.Comment; continue; } if (Char.IsDigit(peek())) { sequenceStart = m_pos; m_state = State.Number; isHex = false; isExp = false; continue; } if(Char.IsWhiteSpace(peek())){ m_pos++; continue; } longBracketLength = getLongBracket(); if (longBracketLength != -1) { isComment = false; m_state = State.LongBracketString; sequenceStart = m_pos; continue; } foreach (Tuple<string, LuaTokenType> op in Ops) { if (match(op.Item1)) { LuaToken rst = new LuaToken(op.Item2, System.Text.Encoding.UTF8.GetBytes(op.Item1), m_pos, getLine(m_pos)); m_pos += op.Item1.Length; return rst; } } if (matchChar(IdentifierStart)) { m_state = State.Identifier; sequenceStart = m_pos; continue; } m_pos++; continue; case State.Identifier: if (matchChar(IdentifierChars)) { m_pos++; continue; } Byte[] str = SubArray(m_data,sequenceStart, m_pos - sequenceStart); foreach (Tuple<string, LuaTokenType> kw in Keywords) { if (equal(kw.Item1,str)) { LuaToken rst = new LuaToken(kw.Item2, System.Text.Encoding.UTF8.GetBytes(kw.Item1), sequenceStart, getLine(sequenceStart)); m_state = State.Start; return rst; } } m_state = State.Start; return new LuaToken(LuaTokenType.Identifier, str, sequenceStart,getLine(sequenceStart)); case State.Comment: { longBracketLength = getLongBracket(); if (longBracketLength != -1) { isComment = true; m_state = State.LongBracketString; sequenceStart = m_pos; continue; } while (!match("\n") && !match(EOF)) { m_pos++; } if (match("\n")) m_pos++; m_state = State.Start; continue; } case State.LongBracketString: { int l = -1; while (!match(EOF) ) { l = matchLongBracket(longBracketLength); if (l != -1) { break; } else { m_pos++; } } Byte[] rst; if (l == -1) { //EOF rst = SubArray(m_data,sequenceStart); } else { rst = SubArray(m_data,sequenceStart, m_pos - sequenceStart); m_pos += l; } m_state = State.Start; if (isComment) { continue; } return new LuaToken(LuaTokenType.StringLiteral, rst, sequenceStart,getLine(sequenceStart)); } case State.String: while (!match(EOF) && !match("\"")) { if (match("\\")) { m_pos++; if (match("\"")) { m_pos++; } } else { m_pos++; } } m_state = State.Start; if (match(EOF)) { return new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data,sequenceStart), sequenceStart,getLine(sequenceStart)); } else { m_pos++; return new LuaToken(LuaTokenType.StringLiteral, SubArray(m_data, sequenceStart, m_pos - sequenceStart - 1), sequenceStart, getLine(sequenceStart)); } case State.Number: if (match("0x") || match("0X")) { m_pos += 2; while (matchChar("0123456789abcdefABCDEF")) { m_pos++; } } else { while (Char.IsDigit(peek())) { m_pos++; } if (match(".")) { m_pos++; while (Char.IsDigit(peek())) { m_pos++; } } if (match("e") || match("E")) { m_pos++; if (match("+") || match("-")) { m_pos++; } while (Char.IsDigit(peek())) { m_pos++; } } } m_state = State.Start; return new LuaToken(LuaTokenType.Number, SubArray(m_data, sequenceStart, m_pos - sequenceStart), sequenceStart, getLine(sequenceStart)); default: throw new Exception("unimplemented state!"); } } }