public void UpdatingWhileFileIsGrowing()
{
IFileSystem fs = Substitute.For <IFileSystem>();
MyFileStream stm = Substitute.For <MyFileStream>(new object());
fs.OpenFile("test").Returns(stm);
DateTime time1 = new DateTime(2000, 1, 1);
long size1 = 100;
stm.Length.Returns(size1);
stm.LastWriteTime.Returns(time1);
stm.IsDeleted.Returns(false);
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
Assert.AreEqual(size1, media.DataStream.Length);
DateTime time2 = new DateTime(2000, 2, 2);
long size2 = 200;
stm.Length.Returns(size2);
stm.LastWriteTime.Returns(time2);
stm.IsDeleted.Returns(false);
media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
}
stm.Received(1).Dispose();
}
public void ExceptionInConstructorMustNotLeakStreams()
{
MockRepository rep = new MockRepository();
MyFileStream stm = rep.CreateMock <MyFileStream>(rep);
stm.Dispose();
LastCall.On(stm).Repeat.AtLeastOnce();
Exception ex = new TestException();
Expect.Call(stm.Length).Repeat.Times(0, 1).Throw(ex);
Expect.Call(stm.IsDeleted).Repeat.Times(0, 1).Throw(ex);
Expect.Call(stm.LastWriteTime).Repeat.Times(0, 1).Throw(ex);
IFileSystem fs = rep.CreateMock <IFileSystem>();
Expect.Call(fs.OpenFile("test")).Return(stm);
rep.ReplayAll();
try
{
(new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))).Dispose();
}
catch (TestException)
{
}
rep.VerifyAll();
}
public void ConstructorAndUpdate()
{
MockRepository rep = new MockRepository();
DateTime modifTime = new DateTime(2000, 1, 1);
long size = 100;
MyFileStream stm = rep.CreateMock <MyFileStream>(rep);
Expect.Call(() => stm.Dispose()).Repeat.AtLeastOnce();
Expect.Call(stm.Length).Return(size);
Expect.Call(stm.IsDeleted).Repeat.Any().Return(false);
Expect.Call(stm.LastWriteTime).Repeat.Any().Return(modifTime);
IFileSystem fs = rep.CreateMock <IFileSystem>();
Expect.Call(fs.OpenFile("test")).Return(stm);
rep.ReplayAll();
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
Assert.AreEqual(modifTime, media.LastModified);
Assert.AreEqual(size, media.Size);
}
rep.VerifyAll();
}
public void InitialUpdateWithInvalidFileNameMustResultInException()
{
Assert.Throws <FileNotFoundException>(() =>
{
IFileSystem fs = Substitute.For <IFileSystem>();
fs.OpenFile("test").Returns(_ => throw new FileNotFoundException());
SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"));
});
}
public void UpdatingWhileFileIsGrowing()
{
MockRepository rep = new MockRepository();
IFileSystem fs = rep.CreateMock <IFileSystem>();
MyFileStream stm = rep.CreateMock <MyFileStream>(rep);
Expect.Call(fs.OpenFile("test")).Return(stm);
DateTime time1 = new DateTime(2000, 1, 1);
long size1 = 100;
Expect.Call(stm.Length).Repeat.Any().Return(size1);
Expect.Call(stm.LastWriteTime).Repeat.Any().Return(time1);
Expect.Call(stm.IsDeleted).Repeat.Any().Return(false);
rep.ReplayAll();
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
Assert.AreEqual(size1, media.DataStream.Length);
rep.VerifyAll();
rep.BackToRecordAll();
DateTime time2 = new DateTime(2000, 2, 2);
long size2 = 200;
Expect.Call(stm.Length).Repeat.Any().Return(size2);
Expect.Call(stm.LastWriteTime).Repeat.Any().Return(time2);
Expect.Call(stm.IsDeleted).Repeat.Any().Return(false);
rep.ReplayAll();
media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
rep.VerifyAll();
rep.BackToRecordAll();
stm.Dispose();
LastCall.On(stm).Repeat.AtLeastOnce();
rep.ReplayAll();
}
rep.VerifyAll();
}
public async Task MediaPropertiesMustChangeOnlyAfterUpdate()
{
IFileSystem fs = Substitute.For <IFileSystem>();
MyFileStream stm = Substitute.For <MyFileStream>(new object());
fs.OpenFile("test").Returns(stm);
DateTime time1 = new DateTime(2000, 1, 1);
long size1 = 100;
stm.Length.Returns(size1);
stm.LastWriteTime.Returns(time1);
stm.IsDeleted.Returns(false);
using (SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
// Media properties are the same as stm's ones
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
// Change the properties of stm
DateTime time2 = new DateTime(2000, 2, 2);
long size2 = 200;
stm.Length.Returns(size2);
stm.LastWriteTime.Returns(time2);
stm.IsDeleted.Returns(false);
// Properties have not still changed
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
// This Update should refresh media's properties
await media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
// Subsequent calls change nothing
await media.Update();
await media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
}
stm.Received(1).Dispose();
}
public void InitialUpdateWithInvalidFileNameMustResultInException()
{
Assert.Throws <FileNotFoundException>(() =>
{
MockRepository rep = new MockRepository();
IFileSystem fs = rep.CreateMock <IFileSystem>();
Expect.Call(fs.OpenFile("test")).Throw(new FileNotFoundException());
rep.ReplayAll();
SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"));
rep.VerifyAll();
});
}
public void ConstructorAndUpdate()
{
DateTime modifTime = new DateTime(2000, 1, 1);
long size = 100;
MyFileStream stm = Substitute.For <MyFileStream>(new object());
stm.Length.Returns(size);
stm.IsDeleted.Returns(false);
stm.LastWriteTime.Returns(modifTime);
IFileSystem fs = Substitute.For <IFileSystem>();
fs.OpenFile("test").Returns(stm);
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
Assert.AreEqual(modifTime, media.LastModified);
Assert.AreEqual(size, media.Size);
}
stm.Received(1).Dispose();
}
public void ExceptionInConstructorMustNotLeakStreams()
{
MyFileStream stm = Substitute.For <MyFileStream>(new object());
Exception ex = new TestException();
stm.Length.Returns(callInfo => { throw ex; });
stm.IsDeleted.Returns(callInfo => { throw ex; });
stm.LastWriteTime.Returns(callInfo => { throw ex; });
IFileSystem fs = Substitute.For <IFileSystem>();
fs.OpenFile("test").Returns(stm);
try
{
(new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))).Dispose();
}
catch (TestException)
{
}
stm.Received(1).Dispose();
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
protected override async Task LiveLogListen(CancellationToken stopEvt, LiveLogXMLWriter output)
{
using (ILogMedia media = await SimpleFileMedia.Create(
host.FileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(fileName)))
using (FileSystemWatcher watcher = new FileSystemWatcher(Path.GetDirectoryName(fileName),
Path.GetFileName(fileName)))
using (AutoResetEvent fileChangedEvt = new AutoResetEvent(true))
{
IMessagesSplitter splitter = new MessagesSplitter(
new StreamTextAccess(media.DataStream, Encoding.ASCII, TextStreamPositioningParams.Default),
host.RegexFactory.Create(@"^(?<body>.+)$", ReOptions.Multiline)
);
watcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size;
watcher.Changed += delegate(object sender, FileSystemEventArgs e)
{
fileChangedEvt.Set();
};
//watcher.EnableRaisingEvents = true;
long lastLinePosition = 0;
long lastStreamLength = 0;
WaitHandle[] events = new WaitHandle[] { stopEvt.WaitHandle, fileChangedEvt };
var capture = new TextMessageCapture();
for (; ;)
{
if (WaitHandle.WaitAny(events, 250, false) == 0)
{
break;
}
await media.Update();
if (media.Size == lastStreamLength)
{
continue;
}
lastStreamLength = media.Size;
sizeInBytesStat = lastStreamLength;
DateTime lastModified = media.LastModified;
await splitter.BeginSplittingSession(new FileRange.Range(0, lastStreamLength), lastLinePosition, MessagesParserDirection.Forward);
try
{
for (; ;)
{
if (!await splitter.GetCurrentMessageAndMoveToNextOne(capture))
{
break;
}
lastLinePosition = capture.BeginPosition;
XmlWriter writer = output.BeginWriteMessage(false);
writer.WriteStartElement("m");
writer.WriteAttributeString("d", Listener.FormatDate(lastModified));
writer.WriteString(XmlUtils.RemoveInvalidXMLChars(capture.MessageHeader));
writer.WriteEndElement();
output.EndWriteMessage();
}
}
finally
{
splitter.EndSplittingSession();
}
}
}
}
public void FileDeletedByAnotherProcessAndThenNewFileAppeared()
{
IFileSystem fs = Substitute.For <IFileSystem>();
// Create and init the first stream
long initialSize1 = 100;
DateTime modifTime1 = new DateTime(2000, 3, 4);
MyFileStream stm1 = Substitute.For <MyFileStream>(new object());
stm1.Length.Returns(initialSize1);
stm1.IsDeleted.Returns(false);
stm1.LastWriteTime.Returns(modifTime1);
// Instruct file system to return the first stream
fs.OpenFile("test").Returns(stm1);
MyFileStream stm2 = Substitute.For <MyFileStream>(new object());
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
// Check that media refers to the first stream stm1
Assert.AreEqual(initialSize1, media.DataStream.Length);
Assert.AreEqual(initialSize1, media.Size);
Assert.AreEqual(true, media.IsAvailable);
// Simulate file deletion: Length and LastWriteTime keep returning file properties,
// but IsDeleted now returns "true".
stm1.IsDeleted.Returns(true);
// Factory cannot open the file that has been deleted while being locked
fs.OpenFile("test").Returns(
_ => throw new UnauthorizedAccessException(),
_ => throw new UnauthorizedAccessException(),
_ => stm2
);
// Properties must return previous values as long as Update is not called
Assert.AreEqual(initialSize1, media.Size);
Assert.AreEqual(initialSize1, media.DataStream.Length);
Assert.AreEqual(true, media.IsAvailable);
// This update should detect file deletion and release it
media.Update();
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
stm1.Received(1).Dispose();
// Subsequent Updates should change nothing
media.Update();
media.Update();
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
// Simulate that new file with name "test" appeared
long initialSize2 = 200;
DateTime modifTime2 = new DateTime(2000, 4, 5);
stm2.Length.Returns(initialSize2);
stm2.IsDeleted.Returns(false);
stm2.LastWriteTime.Returns(modifTime2);
// Properties must return previous values as long as Update is not called
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
// This Update will pick up new file
media.Update();
Assert.AreEqual(initialSize2, media.DataStream.Length);
Assert.AreEqual(initialSize2, media.Size);
Assert.AreEqual(true, media.IsAvailable);
// Subsequent Updates should change nothing
media.Update();
media.Update();
Assert.AreEqual(initialSize2, media.Size);
Assert.AreEqual(initialSize2, media.DataStream.Length);
Assert.AreEqual(true, media.IsAvailable);
}
stm2.Received(1).Dispose();
}
public void MediaPropertiesMustChangeOnlyAfterUpdate()
{
MockRepository rep = new MockRepository();
IFileSystem fs = rep.CreateMock <IFileSystem>();
MyFileStream stm = rep.CreateMock <MyFileStream>(rep);
Expect.Call(fs.OpenFile("test")).Return(stm);
DateTime time1 = new DateTime(2000, 1, 1);
long size1 = 100;
Expect.Call(stm.Length).Repeat.Any().Return(size1);
Expect.Call(stm.LastWriteTime).Repeat.Any().Return(time1);
Expect.Call(stm.IsDeleted).Repeat.Any().Return(false);
rep.ReplayAll();
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
// Media properties are the same as stm's ones
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
rep.VerifyAll();
rep.BackToRecordAll();
// Chnage the properties of stm
DateTime time2 = new DateTime(2000, 2, 2);
long size2 = 200;
Expect.Call(stm.Length).Repeat.Any().Return(size2);
Expect.Call(stm.LastWriteTime).Repeat.Any().Return(time2);
Expect.Call(stm.IsDeleted).Repeat.Any().Return(false);
rep.ReplayAll();
// Properties have not still changed
Assert.AreEqual(time1, media.LastModified);
Assert.AreEqual(size1, media.Size);
// This Update should refresh media's properties
media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
// Subsequent calls change nothing
media.Update();
media.Update();
Assert.AreEqual(time2, media.LastModified);
Assert.AreEqual(size2, media.Size);
rep.VerifyAll();
rep.BackToRecordAll();
stm.Dispose();
LastCall.On(stm).Repeat.AtLeastOnce();
rep.ReplayAll();
}
rep.VerifyAll();
}
public void FileDeletedByAnotherProcessAndThenNewFileAppeared()
{
MockRepository rep = new MockRepository();
IFileSystem fs = rep.CreateMock <IFileSystem>();
// Create and init the first stream
long initialSize1 = 100;
DateTime modifTime1 = new DateTime(2000, 3, 4);
MyFileStream stm1 = rep.CreateMock <MyFileStream>(rep);
Expect.Call(stm1.Length).Repeat.Any().Return(initialSize1);
Expect.Call(stm1.IsDeleted).Repeat.Any().Return(false);
Expect.Call(stm1.LastWriteTime).Repeat.Any().Return(modifTime1);
// Instruct file system to return the first stream
Expect.Call(fs.OpenFile("test")).Return(stm1);
rep.ReplayAll();
using (SimpleFileMedia media = new SimpleFileMedia(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")))
{
// Check that media refers to the first stream stm1
Assert.AreEqual(initialSize1, media.DataStream.Length);
Assert.AreEqual(initialSize1, media.Size);
Assert.AreEqual(true, media.IsAvailable);
rep.VerifyAll();
rep.BackToRecordAll();
// Simulate file deletion: Length and LastWriteTime keep returning file properties,
// but IsDeleted now returns "true".
Expect.Call(stm1.Length).Repeat.Any().Return(initialSize1);
Expect.Call(stm1.LastWriteTime).Repeat.Any().Return(modifTime1);
Expect.Call(stm1.IsDeleted).Repeat.Any().Return(true);
// We expect stream stm1 to be released/disposed
stm1.Dispose();
LastCall.On(stm1).Repeat.AtLeastOnce();
// Factory cannot open the file that has been deleted while being locked
Expect.Call(fs.OpenFile("test")).Repeat.Any().Throw(new UnauthorizedAccessException());
rep.ReplayAll();
// Properties must return previous values as long as Update is not called
Assert.AreEqual(initialSize1, media.Size);
Assert.AreEqual(initialSize1, media.DataStream.Length);
Assert.AreEqual(true, media.IsAvailable);
// This update should detect file deletion and release it
media.Update();
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
// Subsequent Updates should change nothing
media.Update();
media.Update();
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
rep.VerifyAll();
rep.BackToRecordAll();
// Simulate that new file with name "test" appeared
long initialSize2 = 200;
DateTime modifTime2 = new DateTime(2000, 4, 5);
MyFileStream stm2 = rep.CreateMock <MyFileStream>(rep);
Expect.Call(stm2.Length).Repeat.Any().Return(initialSize2);
Expect.Call(stm2.IsDeleted).Repeat.Any().Return(false);
Expect.Call(stm2.LastWriteTime).Repeat.Any().Return(modifTime2);
stm2.Dispose();
LastCall.On(stm2).Repeat.AtLeastOnce();
Expect.Call(fs.OpenFile("test")).Return(stm2);
rep.ReplayAll();
// Properties must return previous values as long as Update is not called
Assert.AreEqual(0, media.Size);
Assert.AreEqual(0, media.DataStream.Length);
Assert.AreEqual(false, media.IsAvailable);
// This Update will pick up new file
media.Update();
Assert.AreEqual(initialSize2, media.DataStream.Length);
Assert.AreEqual(initialSize2, media.Size);
Assert.AreEqual(true, media.IsAvailable);
// Subsequent Updates should change nothing
media.Update();
media.Update();
Assert.AreEqual(initialSize2, media.Size);
Assert.AreEqual(initialSize2, media.DataStream.Length);
Assert.AreEqual(true, media.IsAvailable);
}
rep.VerifyAll();
}