//static int appendKeyword(CTX, kKonohaSpace *ks, kArray *tls, int s, int e, kArray *dst, kToken **tkERR)
private int appendKeyword(IList<Token> tokens, int start, int end, IList<Token> tokensDst, out Token errorToken)
{
int next = start; // don't add
Token tk = tokens[start];
if (tk.Type < TokenType.OPERATOR)
{
tk.Keyword = (KeywordType)tk.Type;
}
if (tk.Type == TokenType.SYMBOL)
{
tk.IsResolved(ctx, ks);
}
else if (tk.Type == TokenType.USYMBOL)
{
if (!tk.IsResolved(ctx, ks))
{
throw new NotImplementedException();
//KonohaClass ct = kKonohaSpace_getCT(ks, null/*FIXME*/, tk.Text, tk.Text.Length, TY_unknown);
object ct = null;
if (ct != null)
{
tk.Keyword = KeywordType.Type;
//tk.Type = ct->cid;
}
}
}
else if (tk.Type == TokenType.OPERATOR)
{
if (!tk.IsResolved(ctx, ks))
{
uint errref = ctx.SUGAR_P(ReportLevel.ERR, tk.ULine, 0, "undefined token: {0}", tk.Text);
tk.toERR(this.ctx, errref);
errorToken = tk;
return end;
}
}
else if (tk.Type == TokenType.CODE)
{
tk.Keyword = KeywordType.Brace;
}
if (tk.IsType)
{
while (next + 1 < end)
{
Token tkN = tokens[next + 1];
if (tkN.TopChar != '[')
break;
List<Token> abuf = new List<Token>();
int atop = abuf.Count;
next = makeTree(TokenType.AST_BRANCET, tokens, next + 1, end, ']', abuf, out errorToken);
if (abuf.Count > atop)
{
tk.ResolveType(this.ctx, abuf[atop]);
}
}
}
if (tk.Keyword > KeywordType.Expr)
{
tokensDst.Add(tk);
}
errorToken = null;
return next;
}
// static int lookAheadKeyword(kArray *tls, int s, int e, kToken *rule)
public int lookAheadKeyword(IList<Token> tls, int s, int e, Token rule)
{
int i;
for (i = s; i < e; i++)
{
Token tk = tls[i];
if (rule.Keyword == tk.Keyword)
return i;
}
return -1;
}
bool isUnaryOp(Context ctx, Token tk)
{
Syntax syn = ks.GetSyntax(tk.Keyword);
return syn != null && syn.Op1 != null;
}
static int TokenizeSymbol(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
int pos = tokStart;
string ts = tenv.Source;
while (pos < ts.Length && IsSymbolic(ts[pos]))
++pos;
token = new Token(TokenType.SYMBOL, ts.Substring(tokStart, pos - tokStart), tokStart);
return pos;
}
static int TokenizeUndefined(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
token = null;
return tokStart;
}
static int TokenizeBlock(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
string ts = tenv.Source;
char ch = '\0';
int pos = tokStart + 1;
int level = 1;
FTokenizer[] fmat = tenv.TokenizerMatrix;
token = null;
while (pos < ts.Length)
{
ch = ts[pos];
if (ch == '}')
{
level--;
if (level == 0)
{
token = new Token(TokenType.CODE, ts.Substring(tokStart + 1, pos - 1 - tokStart), tokStart + 1);
return pos + 1;
}
pos++;
}
else if (ch == '{')
{
level++;
pos++;
}
else
{
var f = fmat[(int)charTypeMatrix[ch]];
pos = f(ctx, out token, tenv, pos, null);
}
}
return pos;
}
static int TokenizeSlash(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
string ts = tenv.Source;
if (ts[tokStart + 1] == '/')
{
return TokenizeLine(ctx, out token, tenv, tokStart, thunk);
}
else if (ts[tokStart + 1] == '*')
{
return TokenizeComment(ctx, out token, tenv, tokStart, thunk);
}
return TokenizeOperator(ctx, out token, tenv, tokStart, thunk);
}
static int TokenizeNumber(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
int pos = tokStart;
bool dotAppeared = false;
string ts = tenv.Source;
while (pos < ts.Length)
{
char ch = ts[pos++];
if (ch == '_')
continue; // nothing
if (ch == '.')
{
if (!IsNumChar(ts[pos]))
{
--pos;
break;
}
dotAppeared = true;
continue;
}
if ((ch == 'e' || ch == 'E') && (ts[pos] == '+' || ts[pos] == '-'))
{
pos++;
continue;
}
if (!IsAlphaOrNum(ch))
{
--pos;
break;
}
}
string str = ts.Substring(tokStart, pos - tokStart).Replace("_", "");
if (dotAppeared)
{
token = new Token(TokenType.FLOAT, str, tokStart);
}
else
{
token = new Token(TokenType.INT, str, tokStart);
}
return pos; // next
}
static int TokenizeOneCharOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
token = new Token(TokenType.OPERATOR, tenv.Source.Substring(tokStart, 1), tokStart);
return ++tokStart;
}
static int TokenizeLine(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
string ts = tenv.Source;
int pos = tokStart;
while (ts[pos] != '\n' || ts[pos] != '\r')
++pos;
token = null;
return pos;
}
static int TokenizeNextline(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
int pos = tokStart;
string ts = tenv.Source;
if (pos < ts.Length)
{
if (ts[pos] == '\r')
{
++pos;
}
if (ts[pos] == '\n')
{
++pos;
}
}
tenv.Line.LineNumber += 1;
tenv.Bol = pos;
return TokenizeIndent(ctx, out token, tenv, pos, thunk);
}
static int TokenizeIndent(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
int pos = tokStart;
string ts = tenv.Source;
int indent = 0;
if (pos < ts.Length)
{
char ch = ts[pos++];
if (ch == '\t')
{
indent += tenv.TabWidth;
}
else if (ch == ' ')
{
indent += 1;
}
else
{
--pos;
}
}
token = null;
return pos;
}
static int TokenizeDoubleQuote(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
string ts = tenv.Source;
char ch = '\0';
char prev = '"';
int pos = tokStart + 1;
token = null;
while (pos < ts.Length)
{
ch = ts[pos++];
if (ch == '\n' || ch == '\r')
{
break;
}
if (ch == '"' && prev != '\\')
{
token = new Token(TokenType.TEXT, ts.Substring(tokStart + 1, (pos - 1) - (tokStart + 1)), tokStart + 1);
return pos;
}
prev = ch;
}
return pos - 1;
}
static int TokenizeComment(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
string ts = tenv.Source;
int pos = tokStart + 2;
char ch = '\0';
char prev = '\0';
int level = 1;
token = null;
while (pos < ts.Length)
{
ch = ts[pos++];
if (ch == '\r')
{
tenv.Line.LineNumber += 1;
if (ts[pos] == '\n')
++pos;
}
else if (ch == '\n')
{
tenv.Line.LineNumber += 1;
}
if (prev == '*' && ch == '/')
{
level--;
if (level == 0)
return pos;
}
else if (prev == '/' && ch == '*')
{
level++;
}
prev = ch;
}
return pos;
}
// static int makeTree(CTX, kKonohaSpace *ks, ktoken_t tt, kArray *tls, int s, int e, int closech, kArray *tlsdst, kToken **tkERRRef)
private int makeTree(TokenType tokentype, IList<Token> tokens, int start, int end, char closeChar, IList<Token> tokensDst, out Token errorToken)
{
int i, probablyCloseBefore = end - 1;
Token tk = tokens[start];
Debug.Assert(tk.Keyword == 0);
Token tkP = new Token(tokentype, tk.Text, closeChar) { Keyword = (KeywordType)tokentype };
tokensDst.Add(tkP);
tkP.Sub = new List<Token>();
for (i = start + 1; i < end; i++)
{
tk = tokens[i];
Debug.Assert(tk.Keyword == 0);
if (tk.Type == TokenType.ERR)
break; // ERR
Debug.Assert(tk.TopChar != '{');
if (tk.TopChar == '(')
{
i = makeTree(TokenType.AST_PARENTHESIS, tokens, i, end, ')', tkP.Sub, out errorToken);
tk.Keyword = KeywordType.Parenthesis;
continue;
}
else if (tk.TopChar == '[')
{
i = makeTree(TokenType.AST_BRANCET, tokens, i, end, ']', tkP.Sub, out errorToken);
tk.Keyword = KeywordType.Brancet;
continue;
}
else if (tk.TopChar == closeChar)
{
errorToken = null;
return i;
}
if ((closeChar == ')' || closeChar == ']') && tk.Type == TokenType.CODE)
probablyCloseBefore = i;
if (tk.Type == TokenType.INDENT && closeChar != '}')
continue; // remove INDENT;
i = appendKeyword(tokens, i, end, tkP.Sub, out errorToken);
}
if (tk.Type != TokenType.ERR)
{
uint errref = ctx.SUGAR_P(ReportLevel.ERR, tk.ULine, 0, "'{0}' is expected (probably before {1})", closeChar.ToString(), tokens[probablyCloseBefore].Text);
tkP.toERR(this.ctx, errref);
}
else
{
tkP.Type = TokenType.ERR;
}
errorToken = tkP;
return end;
}
static int TokenizeOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
{
int pos = tokStart;
string ts = tenv.Source;
while (pos < ts.Length)
{
switch (ts[pos])
{
case '<':
case '>':
case '@':
case '$':
case '#':
case '+':
case '-':
case '*':
case '%':
case '/':
case '=':
case '&':
case '?':
case ':':
case '.':
case '^':
case '!':
case '~':
case '|':
++pos;
continue;
}
break;
}
token = new Token(TokenType.OPERATOR, ts.Substring(tokStart, pos - tokStart), tokStart);
return pos;
}
// static int selectStmtLine(CTX, kKonohaSpace *ks, int *indent, kArray *tls, int s, int e, int delim, kArray *tlsdst, kToken **tkERRRef)
private int SelectStatementLine(ref int indent, IList<Token> tokens, int start, int end, char delim, IList<Token> tokensDst, out Token errorToken)
{
int i = start;
Debug.Assert(end <= tokens.Count);
for (; i < end - 1; i++)
{
Token tk = tokens[i];
Token tk1 = tokens[i + 1];
if (tk.Keyword != 0)
break; // already parsed
if (tk.TopChar == '@' && (tk1.Type == TokenType.SYMBOL || tk1.Type == TokenType.USYMBOL))
{
tk1.Type = TokenType.METANAME;
tk1.Keyword = 0;
tokensDst.Add(tk1);
i++;
if (i + 1 < end && tokens[i + 1].TopChar == '(')
{
i = makeTree(TokenType.AST_PARENTHESIS, tokens, i + 1, end, ')', tokensDst, out errorToken);
}
continue;
}
if (tk.Type == TokenType.METANAME)
{ // already parsed
tokensDst.Add(tk);
if (tk1.Type == TokenType.AST_PARENTHESIS)
{
tokensDst.Add(tk1);
i++;
}
continue;
}
if (tk.Type != TokenType.INDENT)
break;
if (indent == 0)
indent = tk.Text.Length;
}
for (; i < end; i++)
{
var tk = tokens[i];
if (tk.TopChar == delim && tk.Type == TokenType.OPERATOR)
{
errorToken = null;
return i + 1;
}
if (tk.Keyword != 0)
{
tokensDst.Add(tk);
continue;
}
else if (tk.TopChar == '(')
{
i = makeTree(TokenType.AST_PARENTHESIS, tokens, i, end, ')', tokensDst, out errorToken);
tk.Keyword = KeywordType.Parenthesis;
continue;
}
else if (tk.TopChar == '[')
{
i = makeTree(TokenType.AST_BRANCET, tokens, i, end, ']', tokensDst, out errorToken);
tk.Keyword = KeywordType.Brancet;
continue;
}
else if (tk.Type == TokenType.ERR)
{
errorToken = tk;
}
if (tk.Type == TokenType.INDENT)
{
if (tk.Text.Length <= indent)
{
Debug.WriteLine(string.Format("tk.Lpos={0}, indent={1}", tk.Text.Length, indent));
errorToken = null;
return i + 1;
}
continue;
}
i = appendKeyword(tokens, i, end, tokensDst, out errorToken);
}
errorToken = null;
return i;
}
internal Token ResolveType(Context ctx, Token tkP)
{
int i;
// 型引数の取得
int size = tkP.Sub.Count;
var p = new List<KonohaParam>();
for (i = 0; i < size; i++)
{
Token tkT = (tkP.Sub[i]);
if (tkT.Keyword == KeywordType.Type)
{
p.Add(new KonohaParam() { Type = tkT.Type });
}
}
throw new NotImplementedException();
// 以下未実装
KonohaClass ct;
if (p.Count > 0)
{
ct = null;// this.ctx.share.ca.cts[(int)this.Type];
if (ct.cparam == KonohaParam.NULL)
{
ctx.SUGAR_P(ReportLevel.ERR, this.ULine, 0, "not generic type: %s", this.KType.ToString());
return this;
}
//ct = kClassTable_Generics(ct, TY_void, p.Count, p);
}
else
{
//ct = CT_P0(_ctx, CT_Array, this_type(this));
}
this.Type = (TokenType)ct.cid;
return this;
}