/// <summary> /// Responsible for parsing a <see cref="DateToken"/> /// </summary> /// <param name="token">The token</param> /// <returns>Whether the parse was successful</returns> protected virtual bool ParseDateToken(DateToken token) { if (State.ParsedDateState.Date is not null) { return(false); } State.ParsedDateState.Date = token.Value; return(true); }
public void QuickStart() { List <Token> tokens = new List <Token>(); DateToken dateToken = null; Debug.WriteLine("Latvian.Tests.Tokenization.ReadmeTests.QuickStart starts"); string text = "Sveika, pasaule! Man iet labi. Šodienas datums ir 2014-01-01"; LatvianTokenizer tokenizer = new LatvianTokenizer(); foreach (Token token in tokenizer.Tokenize(text)) { Debug.WriteLine("Line {0}: Pos {1}: Type: {2} Token: {3}", token.Line, token.Position, token.GetType(), token.Text); tokens.Add(token); if (token is DateToken) { dateToken = token as DateToken; Debug.WriteLine(dateToken.DateTime.ToString("dd/MM/yyyy")); } } Debug.WriteLine("Latvian.Tests.Tokenization.ReadmeTests.QuickStart end"); Assert.AreEqual(12, tokens.Count); Assert.AreEqual("Sveika", tokens[0].Text); Assert.AreEqual(typeof(WordToken), tokens[0].GetType()); Assert.AreEqual(0, tokens[0].Position); Assert.AreEqual(6, tokens[0].PositionEnd); Assert.AreEqual(0, tokens[0].Line); Assert.AreEqual(0, tokens[0].LineEnd); Assert.AreEqual(0, tokens[0].LinePosition); Assert.AreEqual(6, tokens[0].LinePositionEnd); Assert.AreEqual(",", tokens[1].Text); Assert.AreEqual(typeof(PunctuationToken), tokens[1].GetType()); Assert.AreEqual(6, tokens[1].Position); Assert.AreEqual(7, tokens[1].PositionEnd); Assert.AreEqual(0, tokens[1].Line); Assert.AreEqual(0, tokens[1].LineEnd); Assert.AreEqual(6, tokens[1].LinePosition); Assert.AreEqual(7, tokens[1].LinePositionEnd); Assert.AreEqual("pasaule", tokens[2].Text); Assert.AreEqual(typeof(WordToken), tokens[2].GetType()); Assert.AreEqual(8, tokens[2].Position); Assert.AreEqual(15, tokens[2].PositionEnd); Assert.AreEqual(0, tokens[2].Line); Assert.AreEqual(0, tokens[2].LineEnd); Assert.AreEqual(8, tokens[2].LinePosition); Assert.AreEqual(15, tokens[2].LinePositionEnd); Assert.AreEqual("!", tokens[3].Text); Assert.AreEqual(typeof(PunctuationToken), tokens[3].GetType()); Assert.AreEqual(15, tokens[3].Position); Assert.AreEqual(16, tokens[3].PositionEnd); Assert.AreEqual(0, tokens[3].Line); Assert.AreEqual(0, tokens[3].LineEnd); Assert.AreEqual(15, tokens[3].LinePosition); Assert.AreEqual(16, tokens[3].LinePositionEnd); Assert.AreEqual("Man", tokens[4].Text); Assert.AreEqual(typeof(WordToken), tokens[4].GetType()); Assert.AreEqual(17, tokens[4].Position); Assert.AreEqual(20, tokens[4].PositionEnd); Assert.AreEqual(0, tokens[4].Line); Assert.AreEqual(0, tokens[4].LineEnd); Assert.AreEqual(17, tokens[4].LinePosition); Assert.AreEqual(20, tokens[4].LinePositionEnd); Assert.AreEqual("iet", tokens[5].Text); Assert.AreEqual(typeof(WordToken), tokens[5].GetType()); Assert.AreEqual(21, tokens[5].Position); Assert.AreEqual(24, tokens[5].PositionEnd); Assert.AreEqual(0, tokens[5].Line); Assert.AreEqual(0, tokens[5].LineEnd); Assert.AreEqual(21, tokens[5].LinePosition); Assert.AreEqual(24, tokens[5].LinePositionEnd); Assert.AreEqual("labi", tokens[6].Text); Assert.AreEqual(typeof(WordToken), tokens[6].GetType()); Assert.AreEqual(25, tokens[6].Position); Assert.AreEqual(29, tokens[6].PositionEnd); Assert.AreEqual(0, tokens[6].Line); Assert.AreEqual(0, tokens[6].LineEnd); Assert.AreEqual(25, tokens[6].LinePosition); Assert.AreEqual(29, tokens[6].LinePositionEnd); Assert.AreEqual(".", tokens[7].Text); Assert.AreEqual(typeof(PunctuationToken), tokens[7].GetType()); Assert.AreEqual(29, tokens[7].Position); Assert.AreEqual(30, tokens[7].PositionEnd); Assert.AreEqual(0, tokens[7].Line); Assert.AreEqual(0, tokens[7].LineEnd); Assert.AreEqual(29, tokens[7].LinePosition); Assert.AreEqual(30, tokens[7].LinePositionEnd); Assert.AreEqual("2014-01-01", tokens[11].Text); Assert.AreEqual(typeof(DateToken), tokens[11].GetType()); Assert.AreEqual(50, tokens[11].Position); Assert.AreEqual(60, tokens[11].PositionEnd); Assert.AreEqual(0, tokens[11].Line); Assert.AreEqual(0, tokens[11].LineEnd); Assert.AreEqual(50, tokens[11].LinePosition); Assert.AreEqual(60, tokens[11].LinePositionEnd); Assert.AreEqual(tokens[11], dateToken); Assert.AreEqual("01.01.2014", dateToken.DateTime.ToString("dd/MM/yyyy")); }