public void ShouldRetainTheColumnIndexAfterASnapshotIsCommitted() { var stringTokenizer = new StringTokenizer("1\n2"); stringTokenizer.TakeSnapshot(); stringTokenizer.Consume(); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(2, location.Column); stringTokenizer.CommitSnapshot(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(2, location.Column); }
public void ShouldRestoreTheIndexAfterASnapshotIsRolledback() { var stringTokenizer = new StringTokenizer("1\n2"); stringTokenizer.TakeSnapshot(); stringTokenizer.Consume(); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Index); stringTokenizer.RollbackSnapshot(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(0, location.Index); }
/// <summary> /// Gets the token. /// </summary> /// <param name="tokenizer">The tokenizer.</param> /// <returns> /// A token if the matcher matches successfully; otherwise, null. /// </returns> protected override StringToken GetTokenImpl(StringTokenizer tokenizer) { var location = tokenizer.GetCurrentLocation(); foreach (var character in this.keywordCharacters) { if (tokenizer.Current == character) { tokenizer.Consume(); } else { return(null); } } bool found; if (!this.AllowAsSubString) { var next = tokenizer.Current; found = string.IsNullOrWhiteSpace(next) || this.SpecialCharacters.With(s => s.Any(character => character.IsMatch(tokenizer))); } else { found = true; } return(found ? location.CreateToken(this.TokenType, this.Keyword) : null); }
public void ShouldResetTheColumnIndexAfterANewLineCharacterIsConsumed() { var stringTokenizer = new StringTokenizer("1\n2"); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Line); Assert.AreEqual(1, location.Column); stringTokenizer.Consume(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Line); Assert.AreEqual(2, location.Column); stringTokenizer.Consume(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(2, location.Line); Assert.AreEqual(1, location.Column); }
public void ShouldNotUpdateTheLineNumberUntilAfterADosNewLineCharacterIsConsumed() { var stringTokenizer = new StringTokenizer("1\r\n2"); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Line); stringTokenizer.Consume(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Line); stringTokenizer.Consume(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Line); stringTokenizer.Consume(); location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(2, location.Line); }
public void ShouldBeAbleToGetTheCurrentLocation() { var stringTokenizer = new StringTokenizer("123"); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(0, location.Index); Assert.AreEqual(1, location.Line); Assert.AreEqual(1, location.Column); }
public void ShouldBeAbleToTakeASnapshot() { var stringTokenizer = new StringTokenizer("1\n2"); stringTokenizer.TakeSnapshot(); stringTokenizer.Consume(); var location = stringTokenizer.GetCurrentLocation(); Assert.AreEqual(1, location.Index); Assert.AreEqual(1, location.Line); Assert.AreEqual(2, location.Column); }
/// <summary> /// Gets the token. /// </summary> /// <param name="tokenizer">The tokenizer.</param> /// <returns> /// A token if the matcher matches successfully; otherwise, null. /// </returns> protected override StringToken GetTokenImpl(StringTokenizer tokenizer) { var location = tokenizer.GetCurrentLocation(); bool foundWord = false; var word = new StringBuilder(); while (!tokenizer.IsAtEnd() && !string.IsNullOrWhiteSpace(tokenizer.Current) && !this.SpecialCharacters.With(s => s.Any(c => c.IsMatch(tokenizer)))) { foundWord = true; word.Append(tokenizer.Current); tokenizer.Consume(); } return(foundWord ? location.CreateToken(TokenType.Word, word.ToString()) : null); }
/// <summary> /// Gets the token. /// </summary> /// <param name="tokenizer">The tokenizer.</param> /// <returns> /// A token if the matcher matches successfully; otherwise, null. /// </returns> protected override StringToken GetTokenImpl(StringTokenizer tokenizer) { var location = tokenizer.GetCurrentLocation(); bool foundWhiteSpace = false; var whitesSpace = new StringBuilder(); while (!tokenizer.IsAtEnd() && string.IsNullOrWhiteSpace(tokenizer.Current)) { foundWhiteSpace = true; whitesSpace.Append(tokenizer.Current); tokenizer.Consume(); } if (foundWhiteSpace) { return(location.CreateToken(TokenType.WhiteSpace, whitesSpace.ToString())); } return(null); }
/// <summary> /// Gets the token. /// </summary> /// <param name="tokenizer">The tokenizer.</param> /// <returns> /// A token if the matcher matches successfully; otherwise, null. /// </returns> protected override StringToken GetTokenImpl(StringTokenizer tokenizer) { if (tokenizer.Current == this.StringDelimiter) { var location = tokenizer.GetCurrentLocation(); var str = new StringBuilder(); tokenizer.Consume(); while (!tokenizer.IsAtEnd() && tokenizer.Current != this.StringDelimiter) { str.Append(tokenizer.Current); tokenizer.Consume(); } if (tokenizer.Current == this.StringDelimiter) { tokenizer.Consume(); return(location.CreateToken(TokenType.QuotedString, str.ToString())); } } return(null); }