static IEnumerable <TemplateToken> Parse(string text)
{
var scanner = new TextScanner(text);
while (true)
{
// Find the next start tag
var endOfLastToken = scanner.Position;
var startTag = scanner.ScanTo(StartTag);
if (!startTag.HasValue)
{
// No more start tags found, emit remaining text token
yield return(new TemplateText(text, endOfLastToken, scanner.End));
yield break;
}
// Otherwise emit text between end of last token and the start tag
yield return(new TemplateText(text, endOfLastToken, startTag.Value));
// Find the next end tag
var afterStartTag = scanner.Position;
var endTag = scanner.ScanTo(EndTag);
if (!endTag.HasValue)
{
throw new TemplateParseError("Couldn't find closing '" + EndTag + "'", startTag.Value.ToPosition(text));
}
// emit the text between the start and end tag
yield return(new TemplateVariable(text.Substring(afterStartTag, endTag.Value - afterStartTag)));
}
}
public void CanCheckHasNextLine()
{
string input =
@"First line
4453
Last line";
using (var s = new TextScanner(input))
{
// verified the following with a java sample.
Assert.That(s.HasNextLine(), Is.True);
Assert.That(s.NextLine(), Is.EqualTo("First line"));
Assert.That(s.ToString(), Contains.Substring("position=12"));
Assert.That(s.HasNextLine(), Is.True);
Assert.That(s.NextDouble(), Is.EqualTo(4453.0));
Assert.That(s.ToString(), Contains.Substring("position=16"));
Assert.That(s.HasNextLine(), Is.True);
Assert.That(s.NextLine(), Is.EqualTo(""));
Assert.That(s.ToString(), Contains.Substring("position=18"));
Assert.That(s.HasNextLine(), Is.True);
Assert.That(s.NextLine(), Is.EqualTo("Last line"));
Assert.That(s.HasNextLine(), Is.False);
Assert.Catch(() => s.NextLine());
}
}
protected override void OutputIdentifier(
string identifier,
StringBuilder output,
string sourceSql,
ref int pos)
{
int num = pos;
TextScanner.SkipWhiteSpaces(sourceSql, ref num);
if (TextScanner.Pass('(', sourceSql, ref num))
{
TextScanner.SkipWhiteSpaces(sourceSql, ref num);
if (TextScanner.Pass(')', sourceSql, ref num) && string.Equals(identifier, "GETDATE", StringComparison.InvariantCultureIgnoreCase))
{
pos = num;
output.Append("SYSDATE ");
}
else
{
base.OutputIdentifier(identifier, output, sourceSql, ref pos);
}
}
else if (InDbOracleDatabase.IsReservedWord(identifier))
{
base.OutputIdentifier(identifier, output, sourceSql, ref pos);
}
else
{
this.OutputQualifiedIdentifier(identifier, output, sourceSql, ref pos);
}
}
private static void Main(string[] args)
{
TextScanner s = null;
double sum = 0;
try
{
s = new TextScanner(new StreamReader("usnumbers.txt"));
s.UseCulture(new CultureInfo("en-US"));
while (s.HasNext())
{
if (s.HasNextDouble())
{
sum += s.NextDouble();
}
else
{
s.Next();
}
}
}
finally
{
if (s != null)
{
s.Close();
}
}
Console.WriteLine(sum);
}
public void CanTranslateIndividualTokens()
{
double sum = 0;
using (var s = new TextScanner(new StreamReader("usnumbers.txt")))
{
s.UseCulture(new CultureInfo("en-US"));
int count = 0;
while (s.HasNext())
{
if (s.HasNextDouble())
{
sum += s.NextDouble();
}
else
{
s.Next();
}
Assert.That(count++ < 100);
}
}
Assert.That(sum, Is.EqualTo(1032778.74159));
}
private static void Main(string[] args)
{
TextScanner s = null;
double sum = 0;
try
{
s = new TextScanner(new StreamReader("usnumbers.txt"));
s.UseCulture(new CultureInfo("en-US"));
while (s.HasNext())
{
if (s.HasNextDouble())
{
sum += s.NextDouble();
}
else
{
s.Next();
}
}
}
finally
{
if (s != null)
{
s.Close();
}
}
Console.WriteLine(sum);
}
public void TwoCharactersInLiteralTest()
{
var lexer = Lexer.CreateState("'XX'");
bool result = TextScanner.ScanCharacter(lexer, ref this.token);
Assert.False(result);
}
public void FourBackslashesLiteralTest()
{
var lexer = Lexer.CreateState("'\\\\\\\\'");
bool result = TextScanner.ScanCharacter(lexer, ref this.token);
Assert.False(result);
}
public void NewLineInStringTest()
{
var lexer = Lexer.CreateState("\"\n\"");
bool result = TextScanner.ScanString(lexer, ref this.token);
Assert.False(result);
}
public void TabsInStringTest()
{
var lexer = Lexer.CreateState("\" \"");
bool result = TextScanner.ScanString(lexer, ref this.token);
Assert.True(result);
Assert.Equal("\\t\\t", this.token.Value);
}
private static RequiredDelimitedList CreateTestCase(string input, ILexer <Element> listItemLexer)
{
var lexerFactory = RequiredDelimitedListLexerFactory.Default;
using (var scanner = new TextScanner(new StringTextSource(input)))
{
return(lexerFactory.Create(listItemLexer).Read(scanner));
}
}
public void EscapedDoubleQuoteInStringTest()
{
var lexer = Lexer.CreateState("\"\\\"\"");
bool result = TextScanner.ScanString(lexer, ref this.token);
Assert.True(result);
Assert.Equal("\\\"", this.token.Value);
}
public void CanReadIntegers()
{
var input = "1 3 53\t-1\r\n0";
var expected = new[] { 1, 3, 53, -1, 0 };
using (var s = new TextScanner(input))
{
ScannerEquivalentTest(s, expected, s.HasNextInt32, s.NextInt32);
}
}
public void Read_ShouldSucceed(string input)
{
var lexer = PercentEncodingLexerFactory.Default.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = lexer.Read(scanner);
Assert.Equal(input, result.Text);
}
}
private static void Main(string[] args)
{
using (var s = new TextScanner(new StreamReader("xanadu.txt")))
{
foreach (var token in s)
{
Console.WriteLine(token);
}
}
}
public void Read_ShouldSucceed(string input)
{
var lexer = SegmentNonZeroLengthNoColonsLexerFactory.Default.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = lexer.Read(scanner);
Assert.Equal(input, result.Text);
}
}
private static void Main(string[] args)
{
using (var s = new TextScanner(new StreamReader("xanadu.txt")))
{
foreach (var token in s)
{
Console.WriteLine(token);
}
}
}
protected void PassWhiteSpaces(StringBuilder output, string sourceSql, ref int pos)
{
int startIndex = pos;
TextScanner.SkipWhiteSpaces(sourceSql, ref pos);
if (pos <= startIndex)
{
return;
}
this.OutputWhiteSpaces(sourceSql.Substring(startIndex, pos - startIndex), output, sourceSql, ref pos);
}
private bool containerProhibitsCommas; // frequently during state transitions actions
protected RawTextReader(TextStream input)
{
this.state = GetStateAtContainerStart(IonType.Datagram);
this.valueBuffer = new StringBuilder();
this.scanner = new TextScanner(input);
this.eof = false;
this.valueType = IonType.None;
this.hasNextCalled = false;
this.containerStack = new ContainerStack(this, 6);
this.containerStack.PushContainer(IonType.Datagram);
}
public void CanTranslateAllTokenTypes()
{
string input = "text 9999999999999999999999999999 -32768 2,147,483,647 -9,223,372,036,854,775,808 129 NaN -3.402823e38 true 65,535 4,294,967,295 18,446,744,073,709,551,615 -128";
using (var s = new TextScanner(input)
.UseCulture(new CultureInfo("en-US")))
{
Assert.That(s.HasNext(), Is.True);
Assert.That(s.Next(), Is.EqualTo("text"));
Assert.That(s.HasNextDecimal(), Is.True);
Assert.That(s.NextDecimal(), Is.EqualTo(9999999999999999999999999999m));
Assert.That(s.HasNextInt16(), Is.True);
Assert.That(s.NextInt16(), Is.EqualTo((short)-32768));
Assert.That(s.HasNextInt32(), Is.True);
Assert.That(s.NextInt32(), Is.EqualTo(2147483647));
Assert.That(s.HasNextInt64(), Is.True);
Assert.That(s.NextInt64(), Is.EqualTo(-9223372036854775808L));
Assert.That(s.HasNextByte(), Is.True);
Assert.That(s.NextByte(), Is.EqualTo((byte)129));
Assert.That(s.HasNextDouble(), Is.True);
Assert.That(s.NextDouble(), Is.EqualTo(double.NaN));
Assert.That(s.HasNextSingle(), Is.True);
Assert.That(s.NextSingle(), Is.EqualTo(-3.402823e38f));
Assert.That(s.HasNextBoolean(), Is.True);
Assert.That(s.NextBoolean(), Is.EqualTo(true));
Assert.That(s.HasNextUInt16(), Is.True);
Assert.That(s.NextUInt16(), Is.EqualTo((ushort)65535));
Assert.That(s.HasNextUInt32(), Is.True);
Assert.That(s.NextUInt32(), Is.EqualTo((uint)4294967295U));
Assert.That(s.HasNextUInt64(), Is.True);
Assert.That(s.NextUInt64(), Is.EqualTo((ulong)18446744073709551615U));
Assert.That(s.HasNextSByte(), Is.True);
Assert.That(s.NextSByte(), Is.EqualTo((sbyte)-128));
}
using (var s = new TextScanner("5/1/2008 8:30:52 AM").UseDelimiter(",\\s*"))
{
Assert.That(s.HasNextDateTime(), Is.True);
Assert.That(s.NextDateTime(), Is.EqualTo(new DateTime(2008, 5, 1, 8, 30, 52)));
}
}
public void FromRange(int lowerBound, int upperBound, string encoding, string s)
{
var enc = Encoding.GetEncoding(encoding);
var sut = new ValueRangeLexerFactory();
var lexer = sut.Create(lowerBound, upperBound, enc);
using (var text = new StringTextSource(s))
using (var scanner = new TextScanner(text))
{
var result = lexer.Read(scanner);
Assert.Equal(s, result.Element.Text);
}
}
public void CallingNextAtEndOfLineThrowsException()
{
string input = @"First line, second statement, third statement";
using (var s = new TextScanner(input)
.UseDelimiter(@",\s*"))
{
Assert.That(s.Next(), Is.EqualTo("First line"));
Assert.That(s.NextLine(), Is.EqualTo(", second statement, third statement"));
Assert.Catch <InvalidOperationException>(() => s.Next());
}
}
public void CanReadUppercaseAsciiLetters(string input)
{
var grammar = new CoreGrammar();
grammar.Initialize();
var sut = grammar.Rule("ALPHA");
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = sut.Read(scanner);
Assert.Equal(input, result.Text);
}
}
public void EscapedBackslashLiteralTest()
{
var escaped = new[] { "'\\''", "'\\n'", "'\\r'", "'\\t'", "'\\\\'" };
foreach (string input in escaped)
{
var lexer = Lexer.CreateState(input);
bool result = TextScanner.ScanCharacter(lexer, ref this.token);
Assert.True(result);
}
}
public void FromRange(int lowerBound, int upperBound, string encoding, string s)
{
var enc = Encoding.GetEncoding(encoding);
var sut = new ValueRangeLexerFactory();
var lexer = sut.Create(lowerBound, upperBound, enc);
using (var text = new StringTextSource(s))
using (var scanner = new TextScanner(text))
{
var result = lexer.Read(scanner);
Assert.Equal(s, result.Text);
}
}
public void CanReadUppercaseAsciiLetters(string input)
{
var factory = new AlphaLexerFactory(new ValueRangeLexerFactory(), new AlternativeLexerFactory());
var alphaLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = alphaLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
var factory = new DoubleQuoteLexerFactory(new TerminalLexerFactory());
var lexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = lexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void UnescapedLiteralsTest()
{
var unescaped = new[] { "'\n'", "'\r'", "'\t'", "'\\'", "'\''" };
foreach (string input in unescaped)
{
var lexer = Lexer.CreateState(input);
bool result = TextScanner.ScanCharacter(lexer, ref this.token);
Assert.False(result);
}
}
public void ReadSuccess(string input)
{
var factory = new ControlCharacterLexerFactory(new ValueRangeLexerFactory(), new TerminalLexerFactory(), new AlternativeLexerFactory());
var controlCharacterLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = controlCharacterLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input, string displayName)
{
var factory = new OctetLexerFactory(new ValueRangeLexerFactory());
var octetLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = octetLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
var grammar = new CoreGrammar();
grammar.Initialize();
var sut = grammar.Rule("LWSP");
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = sut.Read(scanner);
Assert.Equal(input, result.Text);
}
}
public void ReadSuccess(string input)
{
var factory = new VisibleCharacterLexerFactory(new ValueRangeLexerFactory());
var visibleCharacterLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = visibleCharacterLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void CanReadUppercaseAsciiLetters(string input)
{
var factory = new AlphaLexerFactory(new ValueRangeLexerFactory(), new AlternativeLexerFactory());
var alphaLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = alphaLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public static void Main()
{
var sw = new StreamWriter(Console.OpenStandardOutput());
sw.NewLine = "\n";
#if DEBUG
sw.AutoFlush = true;
#else
sw.AutoFlush = false;
#endif
Console.SetOut(sw);
_ts = new TextScanner(Console.In);
Solve();
Console.Out.Flush();
}
public void CannotReadNegativeOne()
{
var input = "-1";
var factory = new BitLexerFactory(new AlternativeLexerFactory(), new TerminalLexerFactory());
var bitLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = bitLexer.Read(scanner);
Assert.NotNull(result);
Assert.False(result.Success);
Assert.Null(result.Element);
}
}
public void ReadSuccess(byte input, byte expected)
{
var grammar = new CoreGrammar();
grammar.Initialize();
var sut = grammar.Rule <Octet>("OCTET");
using (var stream = new MemoryStream(new[] { input }))
using (var scanner = new TextScanner(new StreamTextSource(stream)))
{
var result = sut.Read(scanner);
Assert.Equal(expected, result.Value);
}
}
public void ReadSuccess(string input)
{
var factory = new SpaceLexerFactory(new TerminalLexerFactory());
var spaceLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = spaceLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
// General
var terminalLexerFactory = new TerminalLexerFactory();
var alternativeLexerFactory = new AlternativeLexerFactory();
var sequenceLexerFactory = new ConcatenationLexerFactory();
var repetitionLexerFactory = new RepetitionLexerFactory();
// SP
var spaceLexerFactory = new SpaceLexerFactory(terminalLexerFactory);
// HTAB
var horizontalTabLexerFactory = new HorizontalTabLexerFactory(terminalLexerFactory);
// WSP
var whiteSpaceLexerFactory = new WhiteSpaceLexerFactory(
spaceLexerFactory,
horizontalTabLexerFactory,
alternativeLexerFactory);
// CR
var carriageReturnLexerFactory = new CarriageReturnLexerFactory(terminalLexerFactory);
// LF
var lineFeedLexerFactory = new LineFeedLexerFactory(terminalLexerFactory);
// CRLF
var endOfLineLexerFactory = new EndOfLineLexerFactory(
carriageReturnLexerFactory,
lineFeedLexerFactory,
sequenceLexerFactory);
// LWSP
var linearWhiteSpaceLexerFactory = new LinearWhiteSpaceLexerFactory(
whiteSpaceLexerFactory,
endOfLineLexerFactory,
sequenceLexerFactory,
alternativeLexerFactory,
repetitionLexerFactory);
var linearWhiteSpaceLexer = linearWhiteSpaceLexerFactory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = linearWhiteSpaceLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
var terminalLexerFactory = new TerminalLexerFactory();
var spaceLexerFactory = new SpaceLexerFactory(terminalLexerFactory);
var horizontalTabLexerFactory = new HorizontalTabLexerFactory(terminalLexerFactory);
var alternativeLexerFactory = new AlternativeLexerFactory();
var whiteSpaceLexerFactory = new WhiteSpaceLexerFactory(spaceLexerFactory, horizontalTabLexerFactory, alternativeLexerFactory);
var whiteSpaceLexer = whiteSpaceLexerFactory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = whiteSpaceLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
var valueRangeLexerFactory = new ValueRangeLexerFactory();
var digitLexerFactory = new DigitLexerFactory(valueRangeLexerFactory);
var terminalLexerFactory = new TerminalLexerFactory();
var alternativeLexerFactory = new AlternativeLexerFactory();
var factory = new HexadecimalDigitLexerFactory(digitLexerFactory, terminalLexerFactory, alternativeLexerFactory);
var lexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = lexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
public void ReadSuccess(string input)
{
var terminalsLexerFactory = new TerminalLexerFactory();
var sequenceLexerFactory = new ConcatenationLexerFactory();
var carriageReturnLexerFactory = new CarriageReturnLexerFactory(terminalsLexerFactory);
var lineFeedLexerFactory = new LineFeedLexerFactory(terminalsLexerFactory);
var factory = new EndOfLineLexerFactory(carriageReturnLexerFactory, lineFeedLexerFactory, sequenceLexerFactory);
var endOfLineLexer = factory.Create();
using (var scanner = new TextScanner(new StringTextSource(input)))
{
var result = endOfLineLexer.Read(scanner);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.NotNull(result.Element);
Assert.Equal(input, result.Element.Text);
}
}
private static void Main(string[] args)
{
TextScanner s = null;
try
{
s = new TextScanner(new StreamReader("xanadu.txt"));
while (s.HasNext())
{
Console.WriteLine(s.Next());
}
}
finally
{
if (s != null)
{
s.Close();
}
}
}
private TTokenCode token; // code of current token
#endregion Fields
#region Constructors
public StagingParser(TextBuffer buffer, IntermediateCode icode, SymtabStack symbolStack)
{
this.buffer = buffer;
this.icode = icode;
this.symtabStack = symbolStack;
pScanner = new TextScanner(this.buffer);
// -- Enter the special "input" and "output" variable identifiers
// -- into the symbol table.
EnterLocal("input");
EnterLocal("output");
}
