public void TestMethod2() { var Token = new CToken() { Raw = @"""Hello World\n""", Type = CTokenType.String, }; Assert.AreEqual("Hello World\n", Token.GetStringValue()); }
public void ShowLine(CToken TokenCurrent) { var Position = TokenCurrent.Position; var Lines = Text.Split('\n'); Console.Error.WriteLine("{0}", Lines[Position.Row]); //throw new NotImplementedException(); }
public CToken ReadNext() { StartPos = CurrentPos; if (CurrentPos >= Text.Length) { return new CToken() { Position = GetTokenPositionAtCurrentPos(), Raw = "", Type = CTokenType.End, } } ; var Char = Text[CurrentPos]; int CharactersLeft = Text.Length - CurrentPos; var Type = CTokenType.None; bool DoIsNumber = false; if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1]))) { DoIsNumber = true; } // Skip spaces if (IsSpace(Char)) { if (TokenizeSpaces) { if (Char == '\n') { CurrentPos++; Type = CTokenType.NewLine; } else { for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsSpace(Char2) || Char2 == '\n') { break; } } Type = CTokenType.Space; } } else { while (CurrentPos < Text.Length) { Char = Text[CurrentPos]; if (!IsSpace(Char)) { break; } CurrentPos++; } return(ReadNext()); } } // Number? else if (DoIsNumber) { var NumStr = "" + Char; // TODO: support exponent and float numbers. for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsNumberContinuation(Char2)) { break; } NumStr += Char2; } bool IsDouble = (NumStr.Contains(".")); if (NumStr.EndsWith("f")) { IsDouble = true; } if (NumStr.ToLower().StartsWith("0x")) { IsDouble = false; } //Console.WriteLine(NumStr); Type = IsDouble ? CTokenType.Float : CTokenType.Integer; } // Identifier? else if (IsFirstIdentifier(Char)) { for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsAnyIdentifier(Char2)) { break; } } Type = CTokenType.Identifier; } // ' or " else if (Char == '\'' || Char == '"') { bool Closed = false; CurrentPos++; for (; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (Char2 == '\\') { CurrentPos++; continue; } if (Char2 == Char) { Closed = true; break; } } if (!Closed) { throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos)))); } CurrentPos++; Type = (Char == '\'') ? CTokenType.Char : CTokenType.String; } // Operators else { if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3))) { CurrentPos += 3; } else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2))) { CurrentPos += 2; } else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1))) { CurrentPos += 1; } else { throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char))); } Type = CTokenType.Operator; } if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length)) { Console.Error.WriteLine(Text); throw (new Exception("Invalid position")); } var Token = new CToken() { Position = GetTokenPositionAtCurrentPos(), Raw = Text.Substring(StartPos, CurrentPos - StartPos), Type = Type, }; return(Token); }
public CToken ReadNext() { StartPos = CurrentPos; if (CurrentPos >= Text.Length) return new CToken() { Position = GetTokenPositionAtCurrentPos(), Raw = "", Type = CTokenType.End, }; var Char = Text[CurrentPos]; int CharactersLeft = Text.Length - CurrentPos; var Type = CTokenType.None; bool DoIsNumber = false; if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1]))) { DoIsNumber = true; } // Skip spaces if (IsSpace(Char)) { if (TokenizeSpaces) { if (Char == '\n') { CurrentPos++; Type = CTokenType.NewLine; } else { for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsSpace(Char2) || Char2 == '\n') break; } Type = CTokenType.Space; } } else { while (CurrentPos < Text.Length) { Char = Text[CurrentPos]; if (!IsSpace(Char)) break; CurrentPos++; } return ReadNext(); } } // Number? else if (DoIsNumber) { var NumStr = "" + Char; // TODO: support exponent and float numbers. for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsNumberContinuation(Char2)) break; NumStr += Char2; } bool IsDouble = (NumStr.Contains(".")); if (NumStr.EndsWith("f")) IsDouble = true; if (NumStr.ToLower().StartsWith("0x")) IsDouble = false; //Console.WriteLine(NumStr); Type = IsDouble ? CTokenType.Float : CTokenType.Integer; } // Identifier? else if (IsFirstIdentifier(Char)) { for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (!IsAnyIdentifier(Char2)) break; } Type = CTokenType.Identifier; } // ' or " else if (Char == '\'' || Char == '"') { bool Closed = false; CurrentPos++; for (; CurrentPos < Text.Length; CurrentPos++) { var Char2 = Text[CurrentPos]; if (Char2 == '\\') { CurrentPos++; continue; } if (Char2 == Char) { Closed = true; break; } } if (!Closed) throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos)))); CurrentPos++; Type = (Char == '\'') ? CTokenType.Char : CTokenType.String; } // Operators else { if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3))) CurrentPos += 3; else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2))) CurrentPos += 2; else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1))) CurrentPos += 1; else { throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char))); } Type = CTokenType.Operator; } if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length)) { Console.Error.WriteLine(Text); throw (new Exception("Invalid position")); } var Token = new CToken() { Position = GetTokenPositionAtCurrentPos(), Raw = Text.Substring(StartPos, CurrentPos - StartPos), Type = Type, }; return Token; }