public void TestMethod2()
{
var Token = new CToken()
{
Raw = @"""Hello World\n""",
Type = CTokenType.String,
};
Assert.AreEqual("Hello World\n", Token.GetStringValue());
}
public void ShowLine(CToken TokenCurrent)
{
var Position = TokenCurrent.Position;
var Lines = Text.Split('\n');
Console.Error.WriteLine("{0}", Lines[Position.Row]);
//throw new NotImplementedException();
}
public CToken ReadNext()
{
StartPos = CurrentPos;
if (CurrentPos >= Text.Length)
{
return new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = "",
Type = CTokenType.End,
}
}
;
var Char = Text[CurrentPos];
int CharactersLeft = Text.Length - CurrentPos;
var Type = CTokenType.None;
bool DoIsNumber = false;
if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1])))
{
DoIsNumber = true;
}
// Skip spaces
if (IsSpace(Char))
{
if (TokenizeSpaces)
{
if (Char == '\n')
{
CurrentPos++;
Type = CTokenType.NewLine;
}
else
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsSpace(Char2) || Char2 == '\n')
{
break;
}
}
Type = CTokenType.Space;
}
}
else
{
while (CurrentPos < Text.Length)
{
Char = Text[CurrentPos];
if (!IsSpace(Char))
{
break;
}
CurrentPos++;
}
return(ReadNext());
}
}
// Number?
else if (DoIsNumber)
{
var NumStr = "" + Char;
// TODO: support exponent and float numbers.
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsNumberContinuation(Char2))
{
break;
}
NumStr += Char2;
}
bool IsDouble = (NumStr.Contains("."));
if (NumStr.EndsWith("f"))
{
IsDouble = true;
}
if (NumStr.ToLower().StartsWith("0x"))
{
IsDouble = false;
}
//Console.WriteLine(NumStr);
Type = IsDouble ? CTokenType.Float : CTokenType.Integer;
}
// Identifier?
else if (IsFirstIdentifier(Char))
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsAnyIdentifier(Char2))
{
break;
}
}
Type = CTokenType.Identifier;
}
// ' or "
else if (Char == '\'' || Char == '"')
{
bool Closed = false;
CurrentPos++;
for (; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (Char2 == '\\')
{
CurrentPos++; continue;
}
if (Char2 == Char)
{
Closed = true; break;
}
}
if (!Closed)
{
throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos))));
}
CurrentPos++;
Type = (Char == '\'') ? CTokenType.Char : CTokenType.String;
}
// Operators
else
{
if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3)))
{
CurrentPos += 3;
}
else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2)))
{
CurrentPos += 2;
}
else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1)))
{
CurrentPos += 1;
}
else
{
throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char)));
}
Type = CTokenType.Operator;
}
if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length))
{
Console.Error.WriteLine(Text);
throw (new Exception("Invalid position"));
}
var Token = new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = Text.Substring(StartPos, CurrentPos - StartPos),
Type = Type,
};
return(Token);
}
public CToken ReadNext()
{
StartPos = CurrentPos;
if (CurrentPos >= Text.Length) return new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = "",
Type = CTokenType.End,
};
var Char = Text[CurrentPos];
int CharactersLeft = Text.Length - CurrentPos;
var Type = CTokenType.None;
bool DoIsNumber = false;
if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1])))
{
DoIsNumber = true;
}
// Skip spaces
if (IsSpace(Char))
{
if (TokenizeSpaces)
{
if (Char == '\n')
{
CurrentPos++;
Type = CTokenType.NewLine;
}
else
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsSpace(Char2) || Char2 == '\n') break;
}
Type = CTokenType.Space;
}
}
else
{
while (CurrentPos < Text.Length)
{
Char = Text[CurrentPos];
if (!IsSpace(Char)) break;
CurrentPos++;
}
return ReadNext();
}
}
// Number?
else if (DoIsNumber)
{
var NumStr = "" + Char;
// TODO: support exponent and float numbers.
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsNumberContinuation(Char2)) break;
NumStr += Char2;
}
bool IsDouble = (NumStr.Contains("."));
if (NumStr.EndsWith("f")) IsDouble = true;
if (NumStr.ToLower().StartsWith("0x")) IsDouble = false;
//Console.WriteLine(NumStr);
Type = IsDouble ? CTokenType.Float : CTokenType.Integer;
}
// Identifier?
else if (IsFirstIdentifier(Char))
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsAnyIdentifier(Char2)) break;
}
Type = CTokenType.Identifier;
}
// ' or "
else if (Char == '\'' || Char == '"')
{
bool Closed = false;
CurrentPos++;
for (; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (Char2 == '\\') { CurrentPos++; continue; }
if (Char2 == Char) { Closed = true; break; }
}
if (!Closed) throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos))));
CurrentPos++;
Type = (Char == '\'') ? CTokenType.Char : CTokenType.String;
}
// Operators
else
{
if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3))) CurrentPos += 3;
else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2))) CurrentPos += 2;
else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1))) CurrentPos += 1;
else
{
throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char)));
}
Type = CTokenType.Operator;
}
if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length))
{
Console.Error.WriteLine(Text);
throw (new Exception("Invalid position"));
}
var Token = new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = Text.Substring(StartPos, CurrentPos - StartPos),
Type = Type,
};
return Token;
}