private void Process_SingleTokens(string json, JsonToken expected)
{
var tokenizer = new JsonTokenizer();
int tokenCount = tokenizer.Process(json);
Assert.Equal(1, tokenCount);
Assert.Equal(expected, tokenizer.Tokens.Dequeue());
}
private void Process_TokenSets()
{
const int iterations = 10000;
var data = ProcessTestData_SingleTokens
.Select(x => new Tuple <string, JsonToken>((string)x[0], (JsonToken)x[1]))
.ToArray();
var tokenizer = new JsonTokenizer();
var random = new Random(0);
var json = new StringBuilder();
var tokens = new List <JsonToken>();
for (int run = 0; run < iterations; run++)
{
int tokenCount = random.Next(2, 30);
json.Clear();
tokens.Clear();
for (int j = 0; j < tokenCount; j++)
{
int selectedTokenIndex = random.Next(0, data.Length - 1);
json.Append(data[selectedTokenIndex].Item1);
json.Append(WhiteSpaceCharacters[random.Next(0, WhiteSpaceCharacters.Length - 1)]);
tokens.Add(data[selectedTokenIndex].Item2);
}
// --- actual test start ---
int extractedTokenCount = tokenizer.Process(json.ToString());
Assert.Equal(tokens.Count, extractedTokenCount);
Assert.Equal(tokens.ToArray(), tokenizer.Tokens.ToArray());
// --- actual test end ---
tokenizer.Reset();
}
}
/// <summary>
/// Processes the specified JSON string and returns the corresponding log messages
/// (the passed data may contain incomplete JSON documents that are completed over multiple calls).
/// </summary>
/// <param name="data">JSON string to process.</param>
/// <returns>The log messages read from the JSON stream.</returns>
/// <exception cref="JsonMessageReaderException">Reading the log message failed due to a tokenization, parsing or format error.</exception>
public ILogMessage[] Process(string data)
{
mCompletedLogMessages.Clear();
int remainingTokenCount;
try
{
remainingTokenCount = mTokenizer.Process(data, false);
}
catch (TokenizingException ex)
{
// transform exception to ease the interface
throw new JsonMessageReaderException(
ex.LineNumber,
ex.Position,
ex.Message,
ex);
}
while (remainingTokenCount-- > 0)
{
var token = mTokenizer.Tokens.Dequeue();
switch (mState)
{
case State.Start:
{
if (token.Type != JsonTokenType.LBracket)
{
ThrowUnexpectedTokenException(ref token);
}
mStateStack.Push(mState);
mState = State.ReadingObjectKey;
mLogMessage = new LogMessage();
break;
}
case State.ReadingObjectKey:
{
if (token.Type == JsonTokenType.RBracket)
{
// end of object
mState = mStateStack.Pop();
if (mState == State.Start)
{
mCompletedLogMessages.Add(mLogMessage);
}
break;
}
if (token.Type == JsonTokenType.String)
{
// a JSON key
if (token.Token == mFieldNames.Timestamp)
{
mReadingValueState = State.ReadingTimestampValue;
}
else if (token.Token == mFieldNames.HighPrecisionTimestamp)
{
mReadingValueState = State.ReadingHighPrecisionTimestampValue;
}
else if (token.Token == mFieldNames.LogWriter)
{
mReadingValueState = State.ReadingLogWriterValue;
}
else if (token.Token == mFieldNames.LogLevel)
{
mReadingValueState = State.ReadingLogLevelValue;
}
else if (token.Token == mFieldNames.Tags)
{
mReadingValueState = State.ReadingTagsValue;
}
else if (token.Token == mFieldNames.ApplicationName)
{
mReadingValueState = State.ReadingApplicationNameValue;
}
else if (token.Token == mFieldNames.ProcessName)
{
mReadingValueState = State.ReadingProcessNameValue;
}
else if (token.Token == mFieldNames.ProcessId)
{
mReadingValueState = State.ReadingProcessIdValue;
}
else if (token.Token == mFieldNames.Text)
{
mReadingValueState = State.ReadingTextValue;
}
else
{
throw new JsonMessageReaderException(
token.LineNumber,
token.Position,
$"'{token.Token}' at ({token.LineNumber},{token.Position}) is not a valid field name.");
}
mState = State.ExpectingColon;
break;
}
ThrowUnexpectedTokenException(ref token);
break;
}
case State.ExpectingColon:
{
if (token.Type != JsonTokenType.Colon)
{
ThrowUnexpectedTokenException(ref token);
}
mState = mReadingValueState;
break;
}
case State.ExpectingCommaOrEndOfObject:
{
if (token.Type == JsonTokenType.Comma)
{
mState = State.ReadingObjectKey;
break;
}
if (token.Type == JsonTokenType.RBracket)
{
mState = mStateStack.Pop();
if (mState == State.Start)
{
mCompletedLogMessages.Add(mLogMessage);
}
break;
}
ThrowUnexpectedTokenException(ref token);
break;
}
case State.ReadingTimestampValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
if (!DateTimeOffset.TryParseExact(token.Token, TimestampFormat, CultureInfo.InvariantCulture, DateTimeStyles.AllowWhiteSpaces, out var timestamp))
{
throw new JsonMessageReaderException(token.LineNumber, token.Position, $"The timestamp ({token.Token}) does not have the expected format.");
}
mLogMessage.Timestamp = timestamp;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingHighPrecisionTimestampValue:
{
if (token.Type != JsonTokenType.Number)
{
ThrowUnexpectedTokenException(ref token);
}
if (!long.TryParse(token.Token, out long timestamp))
{
throw new JsonMessageReaderException(
token.LineNumber,
token.Position,
$"The high precision timestamp ({token.Token}) does not have the expected format.");
}
mLogMessage.HighPrecisionTimestamp = timestamp;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingLogWriterValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.LogWriterName = token.Token;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingLogLevelValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.LogLevelName = token.Token;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingTagsValue:
{
if (token.Type != JsonTokenType.LSquareBracket)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.Tags = new TagSet();
mState = State.ReadingTagsValueFirstElement;
break;
}
case State.ReadingTagsValueFirstElement:
{
if (token.Type == JsonTokenType.RSquareBracket)
{
// end of tags array
mState = State.ExpectingCommaOrEndOfObject;
break;
}
if (token.Type == JsonTokenType.String)
{
// a tag
mLogMessage.Tags += token.Token;
mState = State.ReadingTagsValueElementDelimiter;
break;
}
ThrowUnexpectedTokenException(ref token);
break;
}
case State.ReadingTagsValueElementDelimiter:
{
if (token.Type == JsonTokenType.RSquareBracket)
{
// end of tags array
mState = State.ExpectingCommaOrEndOfObject;
break;
}
if (token.Type == JsonTokenType.Comma)
{
mState = State.ReadingTagsValueSubsequentElement;
break;
}
ThrowUnexpectedTokenException(ref token);
break;
}
case State.ReadingTagsValueSubsequentElement:
{
if (token.Type == JsonTokenType.String)
{
// a tag
mLogMessage.Tags += token.Token;
mState = State.ReadingTagsValueElementDelimiter;
break;
}
ThrowUnexpectedTokenException(ref token);
break;
}
case State.ReadingApplicationNameValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.ApplicationName = token.Token;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingProcessNameValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.ProcessName = token.Token;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingProcessIdValue:
{
if (token.Type != JsonTokenType.Number)
{
ThrowUnexpectedTokenException(ref token);
}
if (!int.TryParse(token.Token, out int id))
{
throw new JsonMessageReaderException(token.LineNumber, token.Position, $"The process id ({token.Token}) does not have the expected format.");
}
mLogMessage.ProcessId = id;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
case State.ReadingTextValue:
{
if (token.Type != JsonTokenType.String)
{
ThrowUnexpectedTokenException(ref token);
}
mLogMessage.Text = token.Token;
mState = State.ExpectingCommaOrEndOfObject;
break;
}
default:
throw new NotImplementedException();
}
}
return(mCompletedLogMessages.ToArray());
}