public static async Task Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media)))
{
await reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition)), async parser =>
{
for (; ;)
{
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
});
expectation.StartVerification();
for (int i = 0; i < msgs.Count; ++i)
{
expectation.Verify(i, msgs[i]);
}
expectation.FinishVerification();
}
}
public async Task <IReadOnlyList <LogData> > GetLogInformation(Maybe <LogQuery> logQuery)
{
if (logQuery.HasNoValue)
{
return(new List <LogData>());
}
return(await DisposableAsync.Using(() => new DotNetLogEntities("name=DotNetLogEntities" + logQuery.Value.Environment),
context => LogInformation(logQuery, context)));
}
async Task TestSyncResults(FileRange.Range cachedRange, params string[] expectedSyncResults)
{
IPositionedMessagesReader reader = new PositionedMessagesUtilsTests.TestReader(new long[] { 0, 10, 20, 30, 40, 50, 60 });
var(firstMsg, lastMsg) = await PositionedMessagesUtils.GetBoundaryMessages(reader, null);
var availableTime = DateRange.MakeFromBoundaryValues(
firstMsg.Time.ToLocalDateTime(),
lastMsg.Time.ToLocalDateTime()
);
var availableRange = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
var ctx = new CommandContext()
{
Cache = new AsyncLogProviderDataCache()
{
Messages = new LogJoint.MessagesContainers.ListBasedCollection(),
MessagesRange = cachedRange,
},
Stats = new LogProviderStats()
{
PositionsRange = availableRange,
AvailableTime = availableTime,
},
Cancellation = CancellationToken.None,
Preemption = CancellationToken.None,
Reader = reader,
Tracer = LJTraceSource.EmptyTracer
};
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams()
{
Direction = MessagesParserDirection.Forward,
StartPosition = cachedRange.Begin,
Range = cachedRange
}), async parser =>
{
for (; ;)
{
IMessage msg = await parser.ReadNext();
if (msg == null)
{
break;
}
else
{
ctx.Cache.Messages.Add(msg);
}
}
});
await TestSyncResults(ctx, expectedSyncResults);
}
async Task DoTest(LogEntry[] logContent, CreateParserParams originalParams, int jitterBufferSize, LogEntry[] expectedParsedMessages)
{
if (originalParams.Range == null)
{
originalParams.Range = new Range(0, logContent.Length);
}
CreateParserParams validatedParams = originalParams;
validatedParams.EnsureStartPositionIsInRange();
await DisposableAsync.Using(await DejitteringMessagesParser.Create(async p => new ParserImpl(logContent, p), originalParams, jitterBufferSize), async jitter =>
{
int messageIdx;
int idxStep;
if (originalParams.Direction == MessagesParserDirection.Forward)
{
messageIdx = 0;
idxStep = 1;
}
else
{
messageIdx = -1;
idxStep = -1;
}
foreach (LogEntry expectedMessage in expectedParsedMessages)
{
IMessage actualMessage = await jitter.ReadNext();
Assert.IsNotNull(actualMessage);
Assert.AreEqual((long)expectedMessage.Time, actualMessage.Time.ToLocalDateTime().Ticks);
Assert.AreEqual(expectedMessage.Msg, actualMessage.Text.Value);
Assert.AreEqual(validatedParams.StartPosition + messageIdx, actualMessage.Position);
messageIdx += idxStep;
}
IMessage lastMessage = await jitter.ReadNext();
Assert.IsNull(lastMessage);
});
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
