private static Token ContinueReadingComment(StringCodeReader charStream, bool forgive) { char ch; if ((ch = (char)charStream.Read()) != '/') { var e = new UnexpectedCharacterException(charStream.LineNo, charStream.Col - 1, "beginning of any token", ch); PardonOrDeath(e, forgive); return(FailureToken); } var sb = new StringBuilder(); while (true) { var chVal = charStream.Read(); if (chVal < 0) { var e = new UnexpectedCharacterException( charStream.LineNo, charStream.Col, "continuation of comment", (char)0); PardonOrDeath(e, forgive); return(FailureToken); } ch = (char)chVal; if (IsLineSeparator(ch)) { break; } sb.Append(ch); } return(new Token(TokenType.Comment, sb.ToString())); }
private static bool RunTestPair(string input, string expected) { var sb = new StringBuilder(); using (var reader = new StringCodeReader(input)) { int ch; while (true) { ch = reader.Read(); if (ch < 0) { break; } sb.Append((char)ch); } } string got = sb.ToString(); bool ok = String.Equals(expected, got); Trace.WriteLine(ok ? "Test passed:" : "Test failed:\n"); Console.WriteLine("Expecting"); Console.WriteLine(expected); Console.WriteLine("\nGot"); Console.WriteLine(got.Replace("\n", "[\\n]\n")); Console.WriteLine("\n\n"); return(ok); }
private static Token ContinueReadingNakedStringToken(StringCodeReader charStream, char firstChar, bool forgive) { var sb = new StringBuilder(); sb.Append(firstChar); char ch; while (true) { var chVal = charStream.Read(); if (chVal < 0) { var e = new UnexpectedCharacterException( charStream.LineNo, charStream.Col, "continuation of naked string (or decimal)", (char)0); PardonOrDeath(e, forgive); return(FailureToken); } ch = (char)chVal; if (char.IsWhiteSpace((ch))) { break; } sb.Append(ch); } return(new Token(TokenType.NakedString, sb.ToString())); }
/// <summary> /// Parses a regex from a string. /// </summary> /// <param name="pattern">The regex to be parsed.</param> /// <returns></returns> public static GrammarNode <Char> Parse(String pattern) { var reader = new StringCodeReader(pattern); GrammarNode <Char> tree = Parse(reader); if (reader.Position != reader.Length) { throw new RegexParseException(reader.Position, "Input wasn't entirely consumed."); } return(tree); }
private static Token ContinueReadingStringToken(StringCodeReader charStream, bool forgive) { var escaping = false; var openedQuote = false; var sb = new StringBuilder(); while (true) { var chVal = charStream.Read(); if (chVal < 0) { var e = new UnexpectedCharacterException( charStream.LineNo, charStream.Col, "continuation of string literal", (char)0); PardonOrDeath(e, forgive); return(FailureToken); } var ch = (char)chVal; if (escaping) { ReadEscapedChar(ch, sb); escaping = false; } else if (ch == '"') { if (openedQuote) { sb.Append('"'); openedQuote = false; } else { var chPeek = (char)charStream.Peek(); if (char.IsWhiteSpace(chPeek)) { return(new Token(TokenType.StringType, sb.ToString())); } else { openedQuote = true; sb.Append('"'); } } } else if (ch == '\\') { escaping = true; } else { sb.Append(ch); } } }
public void ExecuteExecutesProperly() { var reader = new StringCodeReader("pass"); var tr1 = new Transducer <Char, Int32>(); tr1.InitialState.OnInput(new[] { 'p', 'a', 's', 's' }, 1); Assert.IsTrue(tr1.TryExecute(reader, out var num)); Assert.AreEqual(1, num); reader.Reset(); reader.Advance(1); Assert.IsFalse(tr1.TryExecute(reader, out _)); }
public void FindOffset_Char__ReturnsCorrectValue() { var reader = new StringCodeReader("aaa bbb"); Assert.AreEqual(0, reader.FindOffset('a')); Assert.AreEqual(0, reader.Position); Assert.AreEqual(4, reader.FindOffset('b')); Assert.AreEqual(0, reader.Position); reader.Advance(2); Assert.AreEqual(0, reader.FindOffset('a')); Assert.AreEqual(2, reader.Position); Assert.AreEqual(2, reader.FindOffset('b')); Assert.AreEqual(2, reader.Position); Assert.AreEqual(-1, reader.FindOffset('c')); Assert.AreEqual(2, reader.Position); }
public void FindOffset_String__ReturnsCorrectValue() { var reader = new StringCodeReader("aaa bbb"); Assert.AreEqual(0, reader.FindOffset("aaa")); Assert.AreEqual(0, reader.Position); Assert.AreEqual(4, reader.FindOffset("bbb")); Assert.AreEqual(0, reader.Position); reader.Advance(1); Assert.AreEqual(0, reader.FindOffset("aa")); Assert.AreEqual(1, reader.Position); Assert.AreEqual(-1, reader.FindOffset("aaa")); Assert.AreEqual(1, reader.Position); Assert.AreEqual(3, reader.FindOffset("bbb")); Assert.AreEqual(1, reader.Position); Assert.AreEqual(-1, reader.FindOffset("ccc")); Assert.AreEqual(1, reader.Position); #nullable disable
public void GetLocation__ReturnsCorrectLocation() { var reader = new StringCodeReader("stri\nng\r\nanother\r\nstring"); var expectedLines = new[] { /*s :*/ "B 00 L 1 C 1", /*t :*/ "B 01 L 1 C 2", /*r :*/ "B 02 L 1 C 3", /*i :*/ "B 03 L 1 C 4", /*\n:*/ "B 04 L 1 C 5", /*n :*/ "B 05 L 2 C 1", /*g :*/ "B 06 L 2 C 2", /*\r:*/ "B 07 L 2 C 3", /*\n:*/ "B 08 L 2 C 4", /*a :*/ "B 09 L 3 C 1", /*n :*/ "B 10 L 3 C 2", /*o :*/ "B 11 L 3 C 3", /*t :*/ "B 12 L 3 C 4", /*h :*/ "B 13 L 3 C 5", /*e :*/ "B 14 L 3 C 6", /*r :*/ "B 15 L 3 C 7", /*\r:*/ "B 16 L 3 C 8", /*\n:*/ "B 17 L 3 C 9", /*s :*/ "B 18 L 4 C 1", /*t :*/ "B 19 L 4 C 2", /*r :*/ "B 20 L 4 C 3", /*i :*/ "B 21 L 4 C 4", /*n :*/ "B 22 L 4 C 5", /*g :*/ "B 23 L 4 C 6", }; var i = 0; while (reader.Position != reader.Length) { SourceLocation l = reader.GetLocation(); reader.Advance(1); Assert.AreEqual(expectedLines[i++], $"B {l.Byte:00} L {l.Line} C {l.Column}"); } Assert.ThrowsException <ArgumentOutOfRangeException>(() => reader.Advance(1)); Assert.ThrowsException <ArgumentOutOfRangeException>(() => reader.Advance(-1)); }
public void TestMethod5() { using (var reader = new StringCodeReader("123\n\n 456")) { while (true) { int ch = reader.Peek(); int chr = reader.Read(); Console.Write((ch == chr) ? "1" : String.Format("||{0}|{1}||", ch, chr)); Assert.IsTrue(ch == chr); if (ch < 0 || chr < 0) { break; } } // Getting output? // https://www.codeproject.com/Articles/501610/Getting-Console-Output-Within-A-Unit-Test // No, this. // https://stackoverflow.com/questions/4786884/how-to-write-output-from-a-unit-test#comment100863043_13532856 Console.WriteLine(""); } }
public static Token NextToken(StringCodeReader charStream, bool forgive) { // Assuming no emoji. // May need this later? // https://stackoverflow.com/questions/32895131/how-to-compare-and-convert-emoji-characters-in-c-sharp char ch; do { var chVal = charStream.Read(); if (chVal < 0) { return(EofToken); } ch = (char)chVal; } while (char.IsWhiteSpace(ch)); switch (ch) { case '{': return(LstBeginToken); case '}': return(LstEndToken); case '"': return(ContinueReadingStringToken(charStream, forgive)); case '/': return(ContinueReadingComment(charStream, forgive)); default: if (char.IsLetterOrDigit(ch)) { return(ContinueReadingNakedStringToken(charStream, ch, forgive)); } var e = new UnexpectedCharacterException( charStream.LineNo, charStream.Col, "beginning of any token", ch); PardonOrDeath(e, forgive); return(FailureToken); } }
public void FindOffset_Char_Int32__ReturnsCorrectValue() { var reader = new StringCodeReader("aaa bbb"); Assert.AreEqual(1, reader.FindOffset('a', 1)); Assert.AreEqual(0, reader.Position); Assert.AreEqual(5, reader.FindOffset('b', 5)); Assert.AreEqual(0, reader.Position); Assert.AreEqual(-1, reader.FindOffset('b', 8)); Assert.AreEqual(0, reader.Position); reader.Advance(2); Assert.AreEqual(-1, reader.FindOffset('a', 1)); Assert.AreEqual(2, reader.Position); Assert.AreEqual(3, reader.FindOffset('b', 3)); Assert.AreEqual(2, reader.Position); Assert.AreEqual(-1, reader.FindOffset('c')); Assert.AreEqual(2, reader.Position); Assert.AreEqual(-1, reader.FindOffset('b', 6)); Assert.AreEqual(2, reader.Position); Assert.ThrowsException <ArgumentOutOfRangeException>(() => reader.FindOffset('a', -1), "The offset must be positivel."); }
private AcfFile(string input, bool forgive) { HasError = false; Root = new CompoundNode(null, null); const string ExpectedTokensNameK = "key string or \"}\""; var status = ParserStatus.ExpectingKeyOrRPar; var reader = new StringCodeReader(input); //if (reader.Peek() == 65279) reader.Read(); var stack = new Stack <CompoundNode>(64); var isKeyNaked = false; string lastKey = null; var parent = Root; Token token; Node node; LeafNode lnode, comnode; CompoundNode cnode; ParserException e; do { token = Lexer.NextToken(reader, forgive); if (token.Type == TokenType.Failure) { HasError = true; return; } switch (status) { case ParserStatus.ExpectingKeyOrRPar: switch (token.Type) { case TokenType.StringType: case TokenType.NakedString: lastKey = token.Str; isKeyNaked = token.Type == TokenType.NakedString; status = ParserStatus.ExpectingLParOrVal; break; case TokenType.Comment: comnode = new LeafNode(null, token.Str, parent) { IsComment = true }; parent.Add(comnode); continue; case TokenType.Eof: if (stack.Count == 0) { return; } e = new UnexpectedTokenException( reader.LineNo, reader.Col, ExpectedTokensNameK, token.ToString()); PardonOrDeath(e, forgive); return; case TokenType.LstEnd: if (stack.Count == 0) { e = new UnexpectedTokenException( reader.LineNo, reader.Col, ExpectedTokensNameK, token.ToString()); PardonOrDeath(e, forgive); return; } parent = stack.Pop(); break; default: e = new UnexpectedTokenException( reader.LineNo, reader.Col, ExpectedTokensNameK, token.ToString()); PardonOrDeath(e, forgive); return; } break; case ParserStatus.ExpectingLParOrVal: switch (token.Type) { case TokenType.StringType: case TokenType.NakedString: lnode = new LeafNode(lastKey, token.Str, parent); node = lnode; lnode.IsKeyNaked = isKeyNaked; lnode.IsValueNaked = token.Type == TokenType.NakedString; parent.Add(node); break; case TokenType.Comment: comnode = new LeafNode(null, token.Str, parent) { IsComment = true }; parent.Add(comnode); continue; case TokenType.LstBegin: cnode = new CompoundNode(lastKey, parent) { IsKeyNaked = isKeyNaked }; node = cnode; parent.Add(node); stack.Push(parent); parent = cnode; break; default: e = new UnexpectedTokenException( reader.LineNo, reader.Col, "\"{\" or value string", token.ToString()); PardonOrDeath(e, forgive); return; } status = ParserStatus.ExpectingKeyOrRPar; break; } } while (true); }