public async Task UpdatingWhileFileIsGrowing() { IFileSystem fs = Substitute.For <IFileSystem>(); MyFileStream stm = Substitute.For <MyFileStream>(new object()); fs.OpenFile("test").Returns(stm); DateTime time1 = new DateTime(2000, 1, 1); long size1 = 100; stm.Length.Returns(size1); stm.LastWriteTime.Returns(time1); stm.IsDeleted.Returns(false); using (SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))) { Assert.AreEqual(time1, media.LastModified); Assert.AreEqual(size1, media.Size); Assert.AreEqual(size1, media.DataStream.Length); DateTime time2 = new DateTime(2000, 2, 2); long size2 = 200; stm.Length.Returns(size2); stm.LastWriteTime.Returns(time2); stm.IsDeleted.Returns(false); await media.Update(); Assert.AreEqual(time2, media.LastModified); Assert.AreEqual(size2, media.Size); } stm.Received(1).Dispose(); }
public void InitialUpdateWithInvalidFileNameMustResultInException() { Assert.ThrowsAsync <FileNotFoundException>(async() => { IFileSystem fs = Substitute.For <IFileSystem>(); Task <Stream> throwNotFound() => throw new FileNotFoundException(); fs.OpenFile("test").Returns(_ => throwNotFound()); SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test")); }); }
public async Task MediaPropertiesMustChangeOnlyAfterUpdate() { IFileSystem fs = Substitute.For <IFileSystem>(); MyFileStream stm = Substitute.For <MyFileStream>(new object()); fs.OpenFile("test").Returns(stm); DateTime time1 = new DateTime(2000, 1, 1); long size1 = 100; stm.Length.Returns(size1); stm.LastWriteTime.Returns(time1); stm.IsDeleted.Returns(false); using (SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))) { // Media properties are the same as stm's ones Assert.AreEqual(time1, media.LastModified); Assert.AreEqual(size1, media.Size); // Change the properties of stm DateTime time2 = new DateTime(2000, 2, 2); long size2 = 200; stm.Length.Returns(size2); stm.LastWriteTime.Returns(time2); stm.IsDeleted.Returns(false); // Properties have not still changed Assert.AreEqual(time1, media.LastModified); Assert.AreEqual(size1, media.Size); // This Update should refresh media's properties await media.Update(); Assert.AreEqual(time2, media.LastModified); Assert.AreEqual(size2, media.Size); // Subsequent calls change nothing await media.Update(); await media.Update(); Assert.AreEqual(time2, media.LastModified); Assert.AreEqual(size2, media.Size); } stm.Received(1).Dispose(); }
public async Task ConstructorAndUpdate() { DateTime modifTime = new DateTime(2000, 1, 1); long size = 100; MyFileStream stm = Substitute.For <MyFileStream>(new object()); stm.Length.Returns(size); stm.IsDeleted.Returns(false); stm.LastWriteTime.Returns(modifTime); IFileSystem fs = Substitute.For <IFileSystem>(); fs.OpenFile("test").Returns(stm); using (SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))) { Assert.AreEqual(modifTime, media.LastModified); Assert.AreEqual(size, media.Size); } stm.Received(1).Dispose(); }
public async Task ExceptionInConstructorMustNotLeakStreams() { MyFileStream stm = Substitute.For <MyFileStream>(new object()); Exception ex = new TestException(); stm.Length.Returns(callInfo => { throw ex; }); stm.IsDeleted.Returns(callInfo => { throw ex; }); stm.LastWriteTime.Returns(callInfo => { throw ex; }); IFileSystem fs = Substitute.For <IFileSystem>(); fs.OpenFile("test").Returns(stm); try { (await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))).Dispose(); } catch (TestException) { } stm.Received(1).Dispose(); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); string factoryName = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); Action <double?> setStepDescription = prctComplete => { var str = new StringBuilder(); str.Append(@params.FullPath); str.Append(": fixing timestamp anomalies..."); if (prctComplete != null) { str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100)); } callback.SetStepDescription(str.ToString()); }; setStepDescription(null); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var factoryNameSplit = factoryName.Split('\\'); if (factoryNameSplit.Length != 2) { throw new InvalidFormatException(); } var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]); if (factory == null) { throw new InvalidDataException("factory not found: " + factoryName); } var readerFactory = factory as IMediaBasedReaderFactory; if (readerFactory == null) { throw new InvalidDataException("bad factory: " + factoryName); } using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location))) using (ILogSourceThreadsInternal threads = new LogSourceThreads()) using (var reader = readerFactory.CreateMessagesReader( new MediaBasedReaderParams(threads, fileMedia))) { var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface. if (readerImpl == null) { throw new InvalidDataException("bad reader was made by factory " + factoryName); } await reader.UpdateAvailableBounds(false); var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition); double rangeLen = range.Length; using (var progress = progressAggregator.CreateProgressSink()) using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding)) await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition, flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser => { var queue = new VCSKicksCollection.PriorityQueue <IMessage>( new MessagesComparer(ignoreConnectionIds: true)); Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString()); double lastPrctComplete = 0; var cancellation = callback.Cancellation; for (long msgIdx = 0;; ++msgIdx) { if (cancellation.IsCancellationRequested) { break; } var msg = await parser.ReadNext(); if (msg == null) { break; } if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0) { var prctComplete = (double)(msg.Position - range.Begin) / rangeLen; progress.SetValue(prctComplete); if (prctComplete - lastPrctComplete > 0.05) { setStepDescription(prctComplete); lastPrctComplete = prctComplete; } } queue.Enqueue(msg); if (queue.Count > queueSize) { dequeue(); } } while (queue.Count > 0) { dequeue(); } }); } return(new PreprocessingStepParams( tmpFileName, @params.FullPath + " (reordered)", @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName)) )); }
protected override async Task LiveLogListen(CancellationToken stopEvt, LiveLogXMLWriter output) { using (ILogMedia media = await SimpleFileMedia.Create( host.FileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(fileName))) using (FileSystemWatcher watcher = new FileSystemWatcher(Path.GetDirectoryName(fileName), Path.GetFileName(fileName))) using (AutoResetEvent fileChangedEvt = new AutoResetEvent(true)) { IMessagesSplitter splitter = new MessagesSplitter( new StreamTextAccess(media.DataStream, Encoding.ASCII, TextStreamPositioningParams.Default), host.RegexFactory.Create(@"^(?<body>.+)$", ReOptions.Multiline) ); watcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size; watcher.Changed += delegate(object sender, FileSystemEventArgs e) { fileChangedEvt.Set(); }; //watcher.EnableRaisingEvents = true; long lastLinePosition = 0; long lastStreamLength = 0; WaitHandle[] events = new WaitHandle[] { stopEvt.WaitHandle, fileChangedEvt }; var capture = new TextMessageCapture(); for (; ;) { if (WaitHandle.WaitAny(events, 250, false) == 0) { break; } await media.Update(); if (media.Size == lastStreamLength) { continue; } lastStreamLength = media.Size; sizeInBytesStat = lastStreamLength; DateTime lastModified = media.LastModified; await splitter.BeginSplittingSession(new FileRange.Range(0, lastStreamLength), lastLinePosition, MessagesParserDirection.Forward); try { for (; ;) { if (!await splitter.GetCurrentMessageAndMoveToNextOne(capture)) { break; } lastLinePosition = capture.BeginPosition; XmlWriter writer = output.BeginWriteMessage(false); writer.WriteStartElement("m"); writer.WriteAttributeString("d", Listener.FormatDate(lastModified)); writer.WriteString(XmlUtils.RemoveInvalidXMLChars(capture.MessageHeader)); writer.WriteEndElement(); output.EndWriteMessage(); } } finally { splitter.EndSplittingSession(); } } } }
public async Task FileDeletedByAnotherProcessAndThenNewFileAppeared() { IFileSystem fs = Substitute.For <IFileSystem>(); // Create and init the first stream long initialSize1 = 100; DateTime modifTime1 = new DateTime(2000, 3, 4); MyFileStream stm1 = Substitute.For <MyFileStream>(new object()); stm1.Length.Returns(initialSize1); stm1.IsDeleted.Returns(false); stm1.LastWriteTime.Returns(modifTime1); // Instruct file system to return the first stream fs.OpenFile("test").Returns(stm1); MyFileStream stm2 = Substitute.For <MyFileStream>(new object()); using (SimpleFileMedia media = await SimpleFileMedia.Create(fs, SimpleFileMedia.CreateConnectionParamsFromFileName("test"))) { // Check that media refers to the first stream stm1 Assert.AreEqual(initialSize1, media.DataStream.Length); Assert.AreEqual(initialSize1, media.Size); Assert.AreEqual(true, media.IsAvailable); // Simulate file deletion: Length and LastWriteTime keep returning file properties, // but IsDeleted now returns "true". stm1.IsDeleted.Returns(true); // Factory cannot open the file that has been deleted while being locked fs.OpenFile("test").Returns( _ => throw new UnauthorizedAccessException(), _ => throw new UnauthorizedAccessException(), _ => stm2 ); // Properties must return previous values as long as Update is not called Assert.AreEqual(initialSize1, media.Size); Assert.AreEqual(initialSize1, media.DataStream.Length); Assert.AreEqual(true, media.IsAvailable); // This update should detect file deletion and release it await media.Update(); Assert.AreEqual(0, media.Size); Assert.AreEqual(0, media.DataStream.Length); Assert.AreEqual(false, media.IsAvailable); stm1.Received(1).Dispose(); // Subsequent Updates should change nothing await media.Update(); await media.Update(); Assert.AreEqual(0, media.Size); Assert.AreEqual(0, media.DataStream.Length); Assert.AreEqual(false, media.IsAvailable); // Simulate that new file with name "test" appeared long initialSize2 = 200; DateTime modifTime2 = new DateTime(2000, 4, 5); stm2.Length.Returns(initialSize2); stm2.IsDeleted.Returns(false); stm2.LastWriteTime.Returns(modifTime2); // Properties must return previous values as long as Update is not called Assert.AreEqual(0, media.Size); Assert.AreEqual(0, media.DataStream.Length); Assert.AreEqual(false, media.IsAvailable); // This Update will pick up new file await media.Update(); Assert.AreEqual(initialSize2, media.DataStream.Length); Assert.AreEqual(initialSize2, media.Size); Assert.AreEqual(true, media.IsAvailable); // Subsequent Updates should change nothing await media.Update(); await media.Update(); Assert.AreEqual(initialSize2, media.Size); Assert.AreEqual(initialSize2, media.DataStream.Length); Assert.AreEqual(true, media.IsAvailable); } stm2.Received(1).Dispose(); }