public string GetTokenString(ETokenType tt)
{
ITokenTemplate token = _templates.Find(t => t.GetTokenType() == tt);
if (token == null)
{
return(string.Empty);
}
return(token.GetText());
}
public CToken[] GetTokens(CSentense inSentense)
{
if (string.IsNullOrEmpty(inSentense.Text))
{
return(new CToken[0]);
}
string line = inSentense.Text;
Tuple <CToken, int> comment_pos = FindComments(inSentense);
CToken comment = comment_pos.Item1;
if (comment != null)
{
line = line.Substring(0, comment_pos.Item2).TrimEnd(' ').TrimEnd('\t');
}
List <CToken> lst = new List <CToken>();
int tab_shift = inSentense.Rank * TAB_LENGTH;
int lnum = inSentense.LineNumber;
int world_pos = 0;
int world_len = 0;
int i = 0;
bool open_qute = false;
while (i < line.Length)
{
bool quote = line[i] == '"';
if (quote)
{
open_qute = !open_qute;
}
bool space = false;
ITokenTemplate tmp = null;
if (!open_qute)
{
space = line[i] == ' ';
tmp = _templates.Find(tt => tt.CheckPassed(line, i));
}
bool world_break = space || quote || tmp != null;
if (world_break)
{
world_len = i - world_pos;
if (world_len > 0)
{
AddWorld(i, world_pos, line, lst, lnum, tab_shift);
}
if (tmp == null)
{
world_pos = i + 1;
}
else
{
world_pos = i + tmp.GetText().Length;
}
}
if (tmp == null)
{
i++;
}
else
{
lst.Add(new CToken(tmp.GetTokenType(), tmp.GetText(), inSentense.LineNumber, i + inSentense.Rank * TAB_LENGTH));
i += tmp.GetText().Length;
}
}
world_len = i - world_pos;
if (world_len > 0)
{
AddWorld(i, world_pos, line, lst, lnum, tab_shift);
}
if (comment != null)
{
lst.Add(comment);
}
return(lst.ToArray());
}
