public void Add(int line, int col_start, int col_end, TkType type, int indent, Tk parent = null)
{
if (line >= m_arr.Count)
{
m_arr.Add(new List <Tk>());
}
m_arr[line].Add(new Tk(line, col_start, col_end, type, indent, parent));
tk_tot++;
}
/// <summary>
/// Transforme cet ExpressionToken en string.
/// </summary>
/// <returns></returns>
public override string ToString()
{
if (TkType == ExpressionTokenType.List || TkType == ExpressionTokenType.GenericParametersList || TkType == ExpressionTokenType.BracketList)
{
StringBuilder children = new StringBuilder();
foreach (ExpressionToken token in ListTokens)
{
children.Append(token.ToString() + ",");
}
return("<type=" + TkType.ToString() + ";content='" + Content.ToString() + "';children={" + children.ToString() + "}>");
}
else
{
return("<type=" + TkType.ToString() + ";content='" + Content.ToString() + "'>");
}
}
public Token(TkType type, string value)
{
Type = type;
Value = value;
Subs = new List <Token>();
}
public Token(TkType type) : this(type, null)
{
}
public override string ToString()
{
return("<type=" + TkType.ToString() + "; content='" + Content + "'");
}
//public void FullScanLine(string text, int linenum) {
// m_token_type = TkType.None;
// m_source = text;
// m_tokenpos_start = 0;
// m_tokenpos_start_next = 0;
// //Debug.WriteLine("FullScanLine" + linenum.ToString());
// while (m_token_type != TkType.EOL) {
// TokenNextGet2(text);
// m_tkm.Add(linenum, m_tokenpos_start, m_tokenpos_start_next-1, m_token_type, m_indent);
// }
//}
//private bool CharNext(char ctest, int pos=1) {
// if (m_tokenpos_start + pos >= m_source.Length)
// return false;
// if (m_source[m_tokenpos_start+pos]==ctest)
// return true;
// return false;
//}
// parses line to TkType
// uses m_source
// sets m_tokenpos_start, m_tokenpos_start_next, m_token_type
public void TokenNextGet2(string linestr)
{
Func <char, int, bool> CharNext = (ctest, pos) => {
if (m_tokenpos_start + pos >= linestr.Length)
{
return(false);
}
if (linestr[m_tokenpos_start + pos] == ctest)
{
return(true);
}
return(false);
};
m_tokenpos_start = m_tokenpos_start_next;
if (m_tokenpos_start_next >= linestr.Length)
{
m_token_type = TkType.EOL;
return;
}
switch (linestr[m_tokenpos_start])
{
case '#':
if (m_state == NSScanState.None)
{
m_token_type = TkType.Comment;
m_tokenpos_start_next = linestr.Length;
return;
}
break;
case '\'':
m_token_type = TkType.CharLit;
// 'a', '\"', 'xAA', '\9', '\32', '\255' ,''=should not compile
if (CharNext('\'', 2)) // 'a'
{
m_tokenpos_start_next = m_tokenpos_start + 3;
}
else if (CharNext('\'', 3)) // '\"'
{
m_tokenpos_start_next = m_tokenpos_start + 4;
}
else if (CharNext('\'', 4)) // 'xAA'
{
m_tokenpos_start_next = m_tokenpos_start + 5;
}
else if (CharNext('\'', 5)) // '\255'
{
m_tokenpos_start_next = m_tokenpos_start + 6;
}
else // unfinished char literal?
{
m_token_type = TkType.Other;
m_tokenpos_start_next = m_tokenpos_start + 1;
}
return;
case '"':
if (m_tokenpos_start + 2 < linestr.Length && linestr.Substring(m_tokenpos_start, 3) == "\"\"\"")
{
m_tokenpos_start_next = m_tokenpos_start + 3;
m_token_type = TkType.StringLitLong;
}
else
{
m_tokenpos_start_next = m_tokenpos_start + 1;
m_token_type = TkType.StringLit;
}
return;
case '{':
if (CharNext('.', 1) && !CharNext('.', 2))
{
m_tokenpos_start_next = m_tokenpos_start + 2;
m_token_type = TkType.CurlyDotLeft;
}
else
{
m_tokenpos_start_next = m_tokenpos_start + 1;
m_token_type = TkType.Punctuation;
}
return;
case '.':
// float literals can't be declared/assigned as .1 (compile error)
if (CharNext('}', 1))
{
m_tokenpos_start_next = m_tokenpos_start + 2;
m_token_type = TkType.CurlyDotRight;
}
else
{
m_tokenpos_start_next = m_tokenpos_start + 1;
m_token_type = TkType.Dot;
}
return;
case '(':
m_token_type = TkType.ParenLeft;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case ')':
m_token_type = TkType.ParenRight;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case ',':
m_token_type = TkType.Comma;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case '*':
m_token_type = TkType.Star;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case ':':
case '=':
case ';':
case '/':
case '<':
case '>':
case '!':
case '-':
case '^':
case '|':
case '%':
case '&':
case '$':
case '@':
case '~':
case '?':
case '+':
m_token_type = TkType.Punctuation;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case ' ':
m_token_type = TkType.Space;
m_tokenpos_start_next = m_tokenpos_start;
while (m_tokenpos_start_next + 1 < linestr.Length && linestr[m_tokenpos_start_next + 1] == ' ')
{
m_tokenpos_start_next++;
}
m_tokenpos_start_next++;
if (m_tokenpos_start == 0)
{
m_indent = m_tokenpos_start_next;
}
return;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
m_token_type = TkType.NumberInt;
m_tokenpos_start_next = m_tokenpos_start;
while (m_tokenpos_start_next + 1 < linestr.Length)
{
if (NSScanner.token_nums.IndexOf(linestr[m_tokenpos_start_next + 1]) != -1)
{
m_tokenpos_start_next++;
}
break;
}
m_tokenpos_start_next++;
return;
case '[':
m_token_type = TkType.BracketLeft;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
case ']':
m_token_type = TkType.BracketRight;
m_tokenpos_start_next = m_tokenpos_start + 1;
return;
}
m_token_type = TkType.Other;
m_tokenpos_start_next = linestr.IndexOfAny(NSScanner.token_delims, m_tokenpos_start);
//if (m_tokenpos_start_next!=-1)
// m_token_delim_prev=linestr[m_tokenpos_start_next];
//else
// m_token_delim_prev='\0';
string token_str;
if (m_tokenpos_start_next == -1)
{
token_str = linestr.Substring(m_tokenpos_start);
m_tokenpos_start_next = linestr.Length;
}
else
{
token_str = linestr.Substring(m_tokenpos_start, (m_tokenpos_start_next - m_tokenpos_start));
}
if (LangConst.keywords.Contains(token_str))
{
m_token_type = TkType.Keyword;
m_token_keyword_prev = token_str.ToLower();
return;
}
if (LangConst.datatypes.Contains(token_str))
{
m_token_type = TkType.DataType;
return;
}
if (LangConst.proctypes.Contains(token_str))
{
m_token_type = TkType.Procedure;
return;
}
//int delim_idx=0;
//while (delim_idx < m_tokens_delims.Count && m_tokens_delims[delim_idx]<m_tokenpos_start)
// delim_idx++;
//while (delim_idx<m_tokens_delims.Count){
// if (linestr[m_tokens_delims[delim_idx]]==' ') { delim_idx++; continue; };
// if (linestr[m_tokens_delims[delim_idx]]=='*') { delim_idx++; continue; };
// if (linestr[m_tokens_delims[delim_idx]]=='(') { m_token_type = TkType.Procedure; break; };
// break;
//}
//int delim_next_pos = linestr.IndexOfAny(NSScanner.token_delims, m_tokenpos_start_next);
//char delim_next_char = '\0';
//if (delim_next_pos != -1) {
// delim_next_char = linestr[delim_next_pos];
//}
//if (delim_next_char=='(' || m_token_keyword_prev=="proc") {
// m_token_type = TkType.Procedure;
// return;
//}
}
public void FullScan()
{
Debug.WriteLine("FullScan");
m_nssource.FullScanTime = System.Environment.TickCount;
m_tkm.Clear();
//m_fullscan=2;
m_indent = 0;
int lines_tot;
int col_last;
m_buffer.GetLastLineIndex(out lines_tot, out col_last);
string line_text;
int line_end_idx;
IVsTextLines tlines = m_buffer as IVsTextLines;
for (int line_loop = 0; line_loop < lines_tot; line_loop++)
{
tlines.GetLengthOfLine(line_loop, out line_end_idx);
tlines.GetLineText(line_loop, 0, line_loop, line_end_idx, out line_text);
//FullScanLine(line_text, line_loop);
line_text = line_text == null ? "": line_text;
m_token_type = TkType.None;
m_tokenpos_start = 0;
m_tokenpos_start_next = 0;
//Debug.WriteLine("FullScanLine" + linenum.ToString());
while (m_token_type != TkType.EOL)
{
TokenNextGet2(line_text);
m_tkm.Add(line_loop, m_tokenpos_start, m_tokenpos_start_next - 1, m_token_type, m_indent);
}
}
for (uint tkloop = 0; tkloop < m_tkm.tk_tot; tkloop++)
{
if (m_tkm[tkloop] != null && m_tkm[tkloop].type == TkType.Other)
{
//uint tkspot=tkloop;
for (uint tkspot = tkloop + 1; tkspot < m_tkm.tk_tot; tkspot++)
{
if (m_tkm[tkspot] == null)
{
break;
}
if (m_tkm[tkspot].type == TkType.ParenLeft)
{
m_tkm[tkloop].type = TkType.Procedure;
break;
}
if (m_tkm[tkspot].type == TkType.Space)
{
continue;
}
if (m_tkm[tkspot].type == TkType.Star)
{
continue;
}
break;
}
}
}
}
public Tk(int iline, int icol_start, int icol_end, TkType itype, int iindent, Tk iparent = null)
{
line = iline; col_start = icol_start; col_end = icol_end; type = itype; indent = iindent; parent = iparent;
}
