public void ShouldRetainTheColumnIndexAfterASnapshotIsCommitted()
{
var stringTokenizer = new StringTokenizer("1\n2");
stringTokenizer.TakeSnapshot();
stringTokenizer.Consume();
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(2, location.Column);
stringTokenizer.CommitSnapshot();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(2, location.Column);
}
public void ShouldRestoreTheIndexAfterASnapshotIsRolledback()
{
var stringTokenizer = new StringTokenizer("1\n2");
stringTokenizer.TakeSnapshot();
stringTokenizer.Consume();
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Index);
stringTokenizer.RollbackSnapshot();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(0, location.Index);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
var location = tokenizer.GetCurrentLocation();
foreach (var character in this.keywordCharacters)
{
if (tokenizer.Current == character)
{
tokenizer.Consume();
}
else
{
return(null);
}
}
bool found;
if (!this.AllowAsSubString)
{
var next = tokenizer.Current;
found = string.IsNullOrWhiteSpace(next) || this.SpecialCharacters.With(s => s.Any(character => character.IsMatch(tokenizer)));
}
else
{
found = true;
}
return(found ? location.CreateToken(this.TokenType, this.Keyword) : null);
}
public void ShouldResetTheColumnIndexAfterANewLineCharacterIsConsumed()
{
var stringTokenizer = new StringTokenizer("1\n2");
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Line);
Assert.AreEqual(1, location.Column);
stringTokenizer.Consume();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Line);
Assert.AreEqual(2, location.Column);
stringTokenizer.Consume();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(2, location.Line);
Assert.AreEqual(1, location.Column);
}
public void ShouldNotUpdateTheLineNumberUntilAfterADosNewLineCharacterIsConsumed()
{
var stringTokenizer = new StringTokenizer("1\r\n2");
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Line);
stringTokenizer.Consume();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Line);
stringTokenizer.Consume();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Line);
stringTokenizer.Consume();
location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(2, location.Line);
}
public void ShouldBeAbleToGetTheCurrentLocation()
{
var stringTokenizer = new StringTokenizer("123");
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(0, location.Index);
Assert.AreEqual(1, location.Line);
Assert.AreEqual(1, location.Column);
}
public void ShouldBeAbleToTakeASnapshot()
{
var stringTokenizer = new StringTokenizer("1\n2");
stringTokenizer.TakeSnapshot();
stringTokenizer.Consume();
var location = stringTokenizer.GetCurrentLocation();
Assert.AreEqual(1, location.Index);
Assert.AreEqual(1, location.Line);
Assert.AreEqual(2, location.Column);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
var location = tokenizer.GetCurrentLocation();
bool foundWord = false;
var word = new StringBuilder();
while (!tokenizer.IsAtEnd() && !string.IsNullOrWhiteSpace(tokenizer.Current) && !this.SpecialCharacters.With(s => s.Any(c => c.IsMatch(tokenizer))))
{
foundWord = true;
word.Append(tokenizer.Current);
tokenizer.Consume();
}
return(foundWord ? location.CreateToken(TokenType.Word, word.ToString()) : null);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
var location = tokenizer.GetCurrentLocation();
bool foundWhiteSpace = false;
var whitesSpace = new StringBuilder();
while (!tokenizer.IsAtEnd() && string.IsNullOrWhiteSpace(tokenizer.Current))
{
foundWhiteSpace = true;
whitesSpace.Append(tokenizer.Current);
tokenizer.Consume();
}
if (foundWhiteSpace)
{
return(location.CreateToken(TokenType.WhiteSpace, whitesSpace.ToString()));
}
return(null);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
if (tokenizer.Current == this.StringDelimiter)
{
var location = tokenizer.GetCurrentLocation();
var str = new StringBuilder();
tokenizer.Consume();
while (!tokenizer.IsAtEnd() && tokenizer.Current != this.StringDelimiter)
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
if (tokenizer.Current == this.StringDelimiter)
{
tokenizer.Consume();
return(location.CreateToken(TokenType.QuotedString, str.ToString()));
}
}
return(null);
}