static IEnumerable <TemplateToken> Parse(string text) { var scanner = new TextScanner(text); while (true) { // Find the next start tag var endOfLastToken = scanner.Position; var startTag = scanner.ScanTo(StartTag); if (!startTag.HasValue) { // No more start tags found, emit remaining text token yield return(new TemplateText(text, endOfLastToken, scanner.End)); yield break; } // Otherwise emit text between end of last token and the start tag yield return(new TemplateText(text, endOfLastToken, startTag.Value)); // Find the next end tag var afterStartTag = scanner.Position; var endTag = scanner.ScanTo(EndTag); if (!endTag.HasValue) { throw new TemplateParseError("Couldn't find closing '" + EndTag + "'", startTag.Value.ToPosition(text)); } // emit the text between the start and end tag yield return(new TemplateVariable(text.Substring(afterStartTag, endTag.Value - afterStartTag))); } }
public void CanCheckHasNextLine() { string input = @"First line 4453 Last line"; using (var s = new TextScanner(input)) { // verified the following with a java sample. Assert.That(s.HasNextLine(), Is.True); Assert.That(s.NextLine(), Is.EqualTo("First line")); Assert.That(s.ToString(), Contains.Substring("position=12")); Assert.That(s.HasNextLine(), Is.True); Assert.That(s.NextDouble(), Is.EqualTo(4453.0)); Assert.That(s.ToString(), Contains.Substring("position=16")); Assert.That(s.HasNextLine(), Is.True); Assert.That(s.NextLine(), Is.EqualTo("")); Assert.That(s.ToString(), Contains.Substring("position=18")); Assert.That(s.HasNextLine(), Is.True); Assert.That(s.NextLine(), Is.EqualTo("Last line")); Assert.That(s.HasNextLine(), Is.False); Assert.Catch(() => s.NextLine()); } }
protected override void OutputIdentifier( string identifier, StringBuilder output, string sourceSql, ref int pos) { int num = pos; TextScanner.SkipWhiteSpaces(sourceSql, ref num); if (TextScanner.Pass('(', sourceSql, ref num)) { TextScanner.SkipWhiteSpaces(sourceSql, ref num); if (TextScanner.Pass(')', sourceSql, ref num) && string.Equals(identifier, "GETDATE", StringComparison.InvariantCultureIgnoreCase)) { pos = num; output.Append("SYSDATE "); } else { base.OutputIdentifier(identifier, output, sourceSql, ref pos); } } else if (InDbOracleDatabase.IsReservedWord(identifier)) { base.OutputIdentifier(identifier, output, sourceSql, ref pos); } else { this.OutputQualifiedIdentifier(identifier, output, sourceSql, ref pos); } }
private static void Main(string[] args) { TextScanner s = null; double sum = 0; try { s = new TextScanner(new StreamReader("usnumbers.txt")); s.UseCulture(new CultureInfo("en-US")); while (s.HasNext()) { if (s.HasNextDouble()) { sum += s.NextDouble(); } else { s.Next(); } } } finally { if (s != null) { s.Close(); } } Console.WriteLine(sum); }
public void CanTranslateIndividualTokens() { double sum = 0; using (var s = new TextScanner(new StreamReader("usnumbers.txt"))) { s.UseCulture(new CultureInfo("en-US")); int count = 0; while (s.HasNext()) { if (s.HasNextDouble()) { sum += s.NextDouble(); } else { s.Next(); } Assert.That(count++ < 100); } } Assert.That(sum, Is.EqualTo(1032778.74159)); }
public void TwoCharactersInLiteralTest() { var lexer = Lexer.CreateState("'XX'"); bool result = TextScanner.ScanCharacter(lexer, ref this.token); Assert.False(result); }
public void FourBackslashesLiteralTest() { var lexer = Lexer.CreateState("'\\\\\\\\'"); bool result = TextScanner.ScanCharacter(lexer, ref this.token); Assert.False(result); }
public void NewLineInStringTest() { var lexer = Lexer.CreateState("\"\n\""); bool result = TextScanner.ScanString(lexer, ref this.token); Assert.False(result); }
public void TabsInStringTest() { var lexer = Lexer.CreateState("\" \""); bool result = TextScanner.ScanString(lexer, ref this.token); Assert.True(result); Assert.Equal("\\t\\t", this.token.Value); }
private static RequiredDelimitedList CreateTestCase(string input, ILexer <Element> listItemLexer) { var lexerFactory = RequiredDelimitedListLexerFactory.Default; using (var scanner = new TextScanner(new StringTextSource(input))) { return(lexerFactory.Create(listItemLexer).Read(scanner)); } }
public void EscapedDoubleQuoteInStringTest() { var lexer = Lexer.CreateState("\"\\\"\""); bool result = TextScanner.ScanString(lexer, ref this.token); Assert.True(result); Assert.Equal("\\\"", this.token.Value); }
public void CanReadIntegers() { var input = "1 3 53\t-1\r\n0"; var expected = new[] { 1, 3, 53, -1, 0 }; using (var s = new TextScanner(input)) { ScannerEquivalentTest(s, expected, s.HasNextInt32, s.NextInt32); } }
public void Read_ShouldSucceed(string input) { var lexer = PercentEncodingLexerFactory.Default.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = lexer.Read(scanner); Assert.Equal(input, result.Text); } }
private static void Main(string[] args) { using (var s = new TextScanner(new StreamReader("xanadu.txt"))) { foreach (var token in s) { Console.WriteLine(token); } } }
public void Read_ShouldSucceed(string input) { var lexer = SegmentNonZeroLengthNoColonsLexerFactory.Default.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = lexer.Read(scanner); Assert.Equal(input, result.Text); } }
protected void PassWhiteSpaces(StringBuilder output, string sourceSql, ref int pos) { int startIndex = pos; TextScanner.SkipWhiteSpaces(sourceSql, ref pos); if (pos <= startIndex) { return; } this.OutputWhiteSpaces(sourceSql.Substring(startIndex, pos - startIndex), output, sourceSql, ref pos); }
private bool containerProhibitsCommas; // frequently during state transitions actions protected RawTextReader(TextStream input) { this.state = GetStateAtContainerStart(IonType.Datagram); this.valueBuffer = new StringBuilder(); this.scanner = new TextScanner(input); this.eof = false; this.valueType = IonType.None; this.hasNextCalled = false; this.containerStack = new ContainerStack(this, 6); this.containerStack.PushContainer(IonType.Datagram); }
public void CanTranslateAllTokenTypes() { string input = "text 9999999999999999999999999999 -32768 2,147,483,647 -9,223,372,036,854,775,808 129 NaN -3.402823e38 true 65,535 4,294,967,295 18,446,744,073,709,551,615 -128"; using (var s = new TextScanner(input) .UseCulture(new CultureInfo("en-US"))) { Assert.That(s.HasNext(), Is.True); Assert.That(s.Next(), Is.EqualTo("text")); Assert.That(s.HasNextDecimal(), Is.True); Assert.That(s.NextDecimal(), Is.EqualTo(9999999999999999999999999999m)); Assert.That(s.HasNextInt16(), Is.True); Assert.That(s.NextInt16(), Is.EqualTo((short)-32768)); Assert.That(s.HasNextInt32(), Is.True); Assert.That(s.NextInt32(), Is.EqualTo(2147483647)); Assert.That(s.HasNextInt64(), Is.True); Assert.That(s.NextInt64(), Is.EqualTo(-9223372036854775808L)); Assert.That(s.HasNextByte(), Is.True); Assert.That(s.NextByte(), Is.EqualTo((byte)129)); Assert.That(s.HasNextDouble(), Is.True); Assert.That(s.NextDouble(), Is.EqualTo(double.NaN)); Assert.That(s.HasNextSingle(), Is.True); Assert.That(s.NextSingle(), Is.EqualTo(-3.402823e38f)); Assert.That(s.HasNextBoolean(), Is.True); Assert.That(s.NextBoolean(), Is.EqualTo(true)); Assert.That(s.HasNextUInt16(), Is.True); Assert.That(s.NextUInt16(), Is.EqualTo((ushort)65535)); Assert.That(s.HasNextUInt32(), Is.True); Assert.That(s.NextUInt32(), Is.EqualTo((uint)4294967295U)); Assert.That(s.HasNextUInt64(), Is.True); Assert.That(s.NextUInt64(), Is.EqualTo((ulong)18446744073709551615U)); Assert.That(s.HasNextSByte(), Is.True); Assert.That(s.NextSByte(), Is.EqualTo((sbyte)-128)); } using (var s = new TextScanner("5/1/2008 8:30:52 AM").UseDelimiter(",\\s*")) { Assert.That(s.HasNextDateTime(), Is.True); Assert.That(s.NextDateTime(), Is.EqualTo(new DateTime(2008, 5, 1, 8, 30, 52))); } }
public void FromRange(int lowerBound, int upperBound, string encoding, string s) { var enc = Encoding.GetEncoding(encoding); var sut = new ValueRangeLexerFactory(); var lexer = sut.Create(lowerBound, upperBound, enc); using (var text = new StringTextSource(s)) using (var scanner = new TextScanner(text)) { var result = lexer.Read(scanner); Assert.Equal(s, result.Element.Text); } }
public void CallingNextAtEndOfLineThrowsException() { string input = @"First line, second statement, third statement"; using (var s = new TextScanner(input) .UseDelimiter(@",\s*")) { Assert.That(s.Next(), Is.EqualTo("First line")); Assert.That(s.NextLine(), Is.EqualTo(", second statement, third statement")); Assert.Catch <InvalidOperationException>(() => s.Next()); } }
public void CanReadUppercaseAsciiLetters(string input) { var grammar = new CoreGrammar(); grammar.Initialize(); var sut = grammar.Rule("ALPHA"); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = sut.Read(scanner); Assert.Equal(input, result.Text); } }
public void EscapedBackslashLiteralTest() { var escaped = new[] { "'\\''", "'\\n'", "'\\r'", "'\\t'", "'\\\\'" }; foreach (string input in escaped) { var lexer = Lexer.CreateState(input); bool result = TextScanner.ScanCharacter(lexer, ref this.token); Assert.True(result); } }
public void FromRange(int lowerBound, int upperBound, string encoding, string s) { var enc = Encoding.GetEncoding(encoding); var sut = new ValueRangeLexerFactory(); var lexer = sut.Create(lowerBound, upperBound, enc); using (var text = new StringTextSource(s)) using (var scanner = new TextScanner(text)) { var result = lexer.Read(scanner); Assert.Equal(s, result.Text); } }
public void CanReadUppercaseAsciiLetters(string input) { var factory = new AlphaLexerFactory(new ValueRangeLexerFactory(), new AlternativeLexerFactory()); var alphaLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = alphaLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { var factory = new DoubleQuoteLexerFactory(new TerminalLexerFactory()); var lexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = lexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void UnescapedLiteralsTest() { var unescaped = new[] { "'\n'", "'\r'", "'\t'", "'\\'", "'\''" }; foreach (string input in unescaped) { var lexer = Lexer.CreateState(input); bool result = TextScanner.ScanCharacter(lexer, ref this.token); Assert.False(result); } }
public void ReadSuccess(string input) { var factory = new ControlCharacterLexerFactory(new ValueRangeLexerFactory(), new TerminalLexerFactory(), new AlternativeLexerFactory()); var controlCharacterLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = controlCharacterLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input, string displayName) { var factory = new OctetLexerFactory(new ValueRangeLexerFactory()); var octetLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = octetLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { var grammar = new CoreGrammar(); grammar.Initialize(); var sut = grammar.Rule("LWSP"); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = sut.Read(scanner); Assert.Equal(input, result.Text); } }
public void ReadSuccess(string input) { var factory = new VisibleCharacterLexerFactory(new ValueRangeLexerFactory()); var visibleCharacterLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = visibleCharacterLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public static void Main() { var sw = new StreamWriter(Console.OpenStandardOutput()); sw.NewLine = "\n"; #if DEBUG sw.AutoFlush = true; #else sw.AutoFlush = false; #endif Console.SetOut(sw); _ts = new TextScanner(Console.In); Solve(); Console.Out.Flush(); }
public void CannotReadNegativeOne() { var input = "-1"; var factory = new BitLexerFactory(new AlternativeLexerFactory(), new TerminalLexerFactory()); var bitLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = bitLexer.Read(scanner); Assert.NotNull(result); Assert.False(result.Success); Assert.Null(result.Element); } }
public void ReadSuccess(byte input, byte expected) { var grammar = new CoreGrammar(); grammar.Initialize(); var sut = grammar.Rule <Octet>("OCTET"); using (var stream = new MemoryStream(new[] { input })) using (var scanner = new TextScanner(new StreamTextSource(stream))) { var result = sut.Read(scanner); Assert.Equal(expected, result.Value); } }
public void ReadSuccess(string input) { var factory = new SpaceLexerFactory(new TerminalLexerFactory()); var spaceLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = spaceLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { // General var terminalLexerFactory = new TerminalLexerFactory(); var alternativeLexerFactory = new AlternativeLexerFactory(); var sequenceLexerFactory = new ConcatenationLexerFactory(); var repetitionLexerFactory = new RepetitionLexerFactory(); // SP var spaceLexerFactory = new SpaceLexerFactory(terminalLexerFactory); // HTAB var horizontalTabLexerFactory = new HorizontalTabLexerFactory(terminalLexerFactory); // WSP var whiteSpaceLexerFactory = new WhiteSpaceLexerFactory( spaceLexerFactory, horizontalTabLexerFactory, alternativeLexerFactory); // CR var carriageReturnLexerFactory = new CarriageReturnLexerFactory(terminalLexerFactory); // LF var lineFeedLexerFactory = new LineFeedLexerFactory(terminalLexerFactory); // CRLF var endOfLineLexerFactory = new EndOfLineLexerFactory( carriageReturnLexerFactory, lineFeedLexerFactory, sequenceLexerFactory); // LWSP var linearWhiteSpaceLexerFactory = new LinearWhiteSpaceLexerFactory( whiteSpaceLexerFactory, endOfLineLexerFactory, sequenceLexerFactory, alternativeLexerFactory, repetitionLexerFactory); var linearWhiteSpaceLexer = linearWhiteSpaceLexerFactory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = linearWhiteSpaceLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { var terminalLexerFactory = new TerminalLexerFactory(); var spaceLexerFactory = new SpaceLexerFactory(terminalLexerFactory); var horizontalTabLexerFactory = new HorizontalTabLexerFactory(terminalLexerFactory); var alternativeLexerFactory = new AlternativeLexerFactory(); var whiteSpaceLexerFactory = new WhiteSpaceLexerFactory(spaceLexerFactory, horizontalTabLexerFactory, alternativeLexerFactory); var whiteSpaceLexer = whiteSpaceLexerFactory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = whiteSpaceLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { var valueRangeLexerFactory = new ValueRangeLexerFactory(); var digitLexerFactory = new DigitLexerFactory(valueRangeLexerFactory); var terminalLexerFactory = new TerminalLexerFactory(); var alternativeLexerFactory = new AlternativeLexerFactory(); var factory = new HexadecimalDigitLexerFactory(digitLexerFactory, terminalLexerFactory, alternativeLexerFactory); var lexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = lexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
public void ReadSuccess(string input) { var terminalsLexerFactory = new TerminalLexerFactory(); var sequenceLexerFactory = new ConcatenationLexerFactory(); var carriageReturnLexerFactory = new CarriageReturnLexerFactory(terminalsLexerFactory); var lineFeedLexerFactory = new LineFeedLexerFactory(terminalsLexerFactory); var factory = new EndOfLineLexerFactory(carriageReturnLexerFactory, lineFeedLexerFactory, sequenceLexerFactory); var endOfLineLexer = factory.Create(); using (var scanner = new TextScanner(new StringTextSource(input))) { var result = endOfLineLexer.Read(scanner); Assert.NotNull(result); Assert.True(result.Success); Assert.NotNull(result.Element); Assert.Equal(input, result.Element.Text); } }
private static void Main(string[] args) { TextScanner s = null; try { s = new TextScanner(new StreamReader("xanadu.txt")); while (s.HasNext()) { Console.WriteLine(s.Next()); } } finally { if (s != null) { s.Close(); } } }
private TTokenCode token; // code of current token #endregion Fields #region Constructors public StagingParser(TextBuffer buffer, IntermediateCode icode, SymtabStack symbolStack) { this.buffer = buffer; this.icode = icode; this.symtabStack = symbolStack; pScanner = new TextScanner(this.buffer); // -- Enter the special "input" and "output" variable identifiers // -- into the symbol table. EnterLocal("input"); EnterLocal("output"); }