public bool FragmentedStreamAlreadySaved(IFragment fragment) { if (fragment.FragmentContainer == null) { return(false); // This is not a fragmented stream } if (fragment is IDataBlock) { if (_handledFragmentContainers.Contains(fragment.FragmentContainer)) { return(true); } _handledFragmentContainers.Add(fragment.FragmentContainer); } else if (fragment is ICodecStream) { ICodecStream codecStream = fragment as ICodecStream; var pair = new Pair <IFragmentContainer, string>(codecStream.DataBlock.FragmentContainer, codecStream.Name); if (_handledFragmentContainerStreams.Contains(pair)) { return(true); } _handledFragmentContainerStreams.Add(pair); } return(false); }
private int SaveItem(object item, IDataReaderPool dataReaderPool, string path, OverallProgressReporter overallProgressReporter, HandledContainers handledContainers, bool createForensicIntegrityLog) { IInputFile inputFile = item as IInputFile; if (inputFile != null) { return(SaveInputFile(inputFile, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog)); } IDataBlock dataBlock = item as IDataBlock; if (dataBlock != null) { return(SaveDataBlock(dataBlock, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog)); } ICodecStream codecStream = item as ICodecStream; if (codecStream != null) { path = string.Format("{0}__{1}", path, ReplaceIllegalPathCharactersByUnderscore(codecStream.Name)); return(SaveCodecStream(codecStream, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog)); } return(0); // TODO: is this an error? }
private static string GetFileName(object item) { IInputFile inputFile = item as IInputFile; if (inputFile != null) { return(Path.GetFileName(inputFile.Name)); } IDataBlock containerStream = item as IDataBlock; if (containerStream != null) { return(Path.GetFileName(containerStream.InputFile.Name)); } ICodecStream codecStream = item as ICodecStream; if (codecStream != null) { return(Path.GetFileName(codecStream.InputFile.Name)); } Debug.Fail("item should be of type IInputFile, IDataBlock or ICodecStream"); return(string.Empty); }
//internal IDataPacket GetData(ICodecStream codecStream) //{ // return GetData(codecStream, new NullProgressReporter()); //} internal IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter) { using (IDataReaderPool dataReaderPool = _createDataReaderPool()) { return(_dataBlockScanner.GetData(codecStream, progressReporter, dataReaderPool)); } }
public void CountNumberOfParts(IEnumerable <object> items) { int codecStreamCount = 0; int containerStreamCount = 0; int uniqueStreamCount = 0; HashSet <IFragmentContainer> handledFragmentContainers = new HashSet <IFragmentContainer>(); HashSet <Pair <IFragmentContainer, string> > handledFragmentContainerStreams = new HashSet <Pair <IFragmentContainer, string> >(); foreach (object item in items) { IInputFile inputFile = item as IInputFile; IDataBlock containerStream = item as IDataBlock; ICodecStream codecStream = item as ICodecStream; if (inputFile != null) { IList <IDataBlock> dataBlocks = inputFile.Project.GetDataBlocks(inputFile); containerStreamCount += dataBlocks.Count; foreach (IDataBlock dataBlock in dataBlocks) { codecStreamCount += dataBlock.CodecStreams.Count; uniqueStreamCount += CountUniqueStreams(handledFragmentContainers, handledFragmentContainerStreams, dataBlock); } } else if (containerStream != null) { containerStreamCount++; codecStreamCount += containerStream.CodecStreams.Count; uniqueStreamCount += CountUniqueStreams(handledFragmentContainers, handledFragmentContainerStreams, containerStream); } else if (codecStream != null) { // Count the number of fragments for this codecStream. // The codec stream can be part of a fragmented codec stream // which on its turn can be part of a fragmented container stream if (codecStream.DataBlock.FragmentContainer != null) { foreach (IDataBlock containerFragment in codecStream.DataBlock.FragmentContainer) { codecStreamCount += CountCodecStreamFragments(containerFragment, codecStream.Name); } } else { codecStreamCount += CountCodecStreamFragments(codecStream.DataBlock, codecStream.Name); } if (FragmentedStreamAlreadySaved(handledFragmentContainers, handledFragmentContainerStreams, codecStream) == false) { uniqueStreamCount++; } } } // Progress is reported for each: // - codec stream (twice, once during DetectData and once during GetDataPacket) // - container stream and codec stream that is saved _partCount = codecStreamCount * 2 + containerStreamCount + uniqueStreamCount; }
public FileScanResult(IFragment sourceFragment, IResultNode result, IDataBlock resultDataBlock, ICodecStream resultCodecStream, int availableCodecStreams) { IsValid = true; _sourceFragment = sourceFragment; _result = result; _resultDataBlock = resultDataBlock; _resultCodecStream = resultCodecStream; _availableCodecStreams = availableCodecStreams; }
/// <summary> /// Send consumer, reading messages one by one from stream and writes /// to websocket stream. /// </summary> /// <returns></returns> protected virtual async Task SendConsumerAsync(ICodecStream <S> codec) { if (_lastMessage == null) { _lastMessage = Message.Create(StreamId, _remoteId, PollRequest.Create((ulong)_timeout.TotalMilliseconds)); } while (!_open.IsCancellationRequested) { try { if (_lastMessage == null) { if (_send.Completion.IsCompleted) { // Pipeline closed, close the connection _receive.Complete(); _open.Cancel(); break; } try { _lastMessage = await _send.ReceiveAsync(_timeout, _open.Token).ConfigureAwait(false); if (_lastMessage.Content is DataMessage data) { data.SequenceNumber = _nextSendSequenceNumber++; } } catch (TimeoutException) { _lastMessage = Message.Create(StreamId, _remoteId, PollRequest.Create((ulong)_timeout.TotalMilliseconds)); } catch (OperationCanceledException) when(!_open.IsCancellationRequested) { _lastMessage = Message.Create(StreamId, _remoteId, PollRequest.Create((ulong)_timeout.TotalMilliseconds)); } } await codec.WriteAsync(_lastMessage, _open.Token).ConfigureAwait(false); await codec.Stream.FlushAsync(_open.Token).ConfigureAwait(false); _lastMessage.Dispose(); _lastMessage = null; } catch (OperationCanceledException) { break; } catch (Exception e) { ProxyEventSource.Log.StreamException(this, codec.Stream, e); break; } } }
public void TearDown() { _mockRepository = null; _codecStream = null; _dataWriter = null; _progressReporter1 = null; _detectors = null; _dataReaderPool = null; _createDataWriter = null; _createSubProgressReporter = null; _saveAsSeparateFiles = null; }
/// <summary> /// Gets the data for the given <paramref name="codecStream"/> and /// reports progress to the given <paramref name="progressReporter"/>. /// This rescans the parent (container) of the codecStream to determine /// the data of codecStream. /// </summary> /// <param name="codecStream">the codec stream to retrieve the data for</param> /// <param name="progressReporter">the progress reporter</param> /// <param name="dataReaderPool">The shared pool of file data readers</param> /// <returns>the root node of the results</returns> public IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter, IDataReaderPool dataReaderPool) { IDataPacket data; if (_dataCache.TryGetValue(codecStream, out data)) { return(new RescannedCodecStream(codecStream, data)); } IDataBlock containerStreamDataBlock = codecStream.DataBlock; using (IDataReader dataReader = new ProgressDataReader(new FragmentedDataReader(containerStreamDataBlock, dataReaderPool), progressReporter)) { IDetector containerDetector = containerStreamDataBlock.Detectors.FirstOrDefault(); IScanContext scanContext = _createScanContext(containerStreamDataBlock.InputFile.Project); scanContext.Detectors = new[] { containerDetector }; IDataBlockBuilder containerStreamDataBlockBuilder = _createDataBlockBuilder(); containerStreamDataBlockBuilder.Detectors = scanContext.Detectors; containerStreamDataBlockBuilder.InputFile = containerStreamDataBlock.InputFile; IDataBlock newContainerStream = containerDetector.DetectData(dataReader, containerStreamDataBlockBuilder, scanContext); // Check cancellation request if (progressReporter.CancellationPending) { return(null); } // Check results / container stream if (newContainerStream == null) { RetrievedDetectableIsDifferentThanTheOneRetrievedDuringTheInitialScan(); return(null); } data = newContainerStream.CodecStreams.Where(c => c.StreamNumber == codecStream.StreamNumber).FirstOrDefault(); if ((data == null) || (data.Length < codecStream.EndOffset)) { RetrievedDetectableIsDifferentThanTheOneRetrievedDuringTheInitialScan(); return(null); } data = data.GetSubPacket(codecStream.StartOffset, codecStream.Length); } CacheData(codecStream, data); return(new RescannedCodecStream(codecStream, data)); }
public void SetUp() { _mockRepository = new MockRepository(); _codecStream = MockRepository.GenerateStub <ICodecStream>(); _dataWriter = _mockRepository.StrictMock <IDataWriter>(); _progressReporter1 = MockRepository.GenerateStub <IProgressReporter>(); _detectors = Enumerable.Empty <IDetector>(); _dataReaderPool = MockRepository.GenerateStub <IDataReaderPool>(); _createDataWriter = MockRepository.GenerateStub <Creator <IDataWriter, string> >(); _createSubProgressReporter = MockRepository.GenerateStub <Creator <IProgressReporter, IProgressReporter, long, long, long> >(); _forensicIntegrityLog = MockRepository.GenerateStub <IForensicIntegrityLog>(); _createDataWriter.Stub(x => x(FileName)).Return(_dataWriter).Repeat.Once(); _saveAsSeparateFiles = new SaveAsSeparateFiles(_createDataWriter, _createSubProgressReporter, TestFramework.DataBlockScanner, _forensicIntegrityLog); }
private IList <ICodecStream> BuildCodecStreams(IDataBlock dataBlock) { List <ICodecStream> codecStreams = new List <ICodecStream>(); IFragment previousFragment = null; foreach (ICodecStreamBuilder codecStreamBuilder in _codecStreamBuilders) { codecStreamBuilder.DataBlock = dataBlock; codecStreamBuilder.PreviousFragment = previousFragment; ICodecStream codecStream = codecStreamBuilder.Build(); codecStreams.Add(codecStream); previousFragment = codecStream; } return(codecStreams.AsReadOnly()); }
internal CompleteCodecStreamScanner(IDataScanner dataScanner, IDataBlockBuilder dataBlockBuilder, ICodecStream completeCodecStream, IProgressReporter progressReporter) { _dataScanner = dataScanner; _dataBlockBuilder = dataBlockBuilder; _completeCodecStream = completeCodecStream; _progressReporter = progressReporter; _codecStreamFragments = new List <IFragment>(); long length = _completeCodecStream.Length; _maxUnknownDataForCompleteStream = Math.Max((MaxPercentageUnkownDataForCompleteStream * length) / 100, Math.Min((2 * MaxUnparsableBytes), length)); long detectBytes = Math.Min(length, MaxBytesForDetect); _maxUnknownDataForDetect = Math.Max((MaxPercentageUnkownDataForDetect * detectBytes) / 100, Math.Min((2 * MaxUnparsableBytes), detectBytes)); }
/// <summary> /// Wrapper that calls dispose on the stream - at a minimum... /// </summary> /// <param name="codec"></param> /// <param name="parentHasFaulted"></param> /// <returns></returns> private async Task CloseStreamAsync(ICodecStream <S> codec, bool parentHasFaulted) { ProxyEventSource.Log.StreamClosing(this, codec.Stream); if (!parentHasFaulted) { await CloseStreamAsync(codec).ConfigureAwait(false); } try { codec.Stream.Dispose(); } catch (Exception e) { ProxyEventSource.Log.HandledExceptionAsInformation(codec.Stream, e); } ProxyEventSource.Log.StreamClosed(this, codec.Stream); }
/// <summary> /// Receive producer, reading messages one by one from relay and /// writing to receive block. Manages stream as well. /// </summary> /// <returns></returns> protected virtual async Task ReceiveProducerAsync(ICodecStream <S> codec) { while (!_open.IsCancellationRequested) { try { // Read message and send to source block var message = await codec.ReadAsync <Message>(_open.Token).ConfigureAwait(false); if (message != null) { if (message.Content is DataMessage data && data.SequenceNumber != _nextReceiveSequenceNumber++) { // TODO: Implement poll for previous message if (data.SequenceNumber > _nextReceiveSequenceNumber - 1) { ProxyEventSource.Log.MissingData(this, data, _nextReceiveSequenceNumber - 1); message.Error = (int)SocketError.Missing; } else { ProxyEventSource.Log.DuplicateData(this, data, _nextReceiveSequenceNumber - 1); // Do not even bother to forward continue; } } if (!await _receive.SendAsync(message, _open.Token).ConfigureAwait(false) || message.TypeId == MessageContent.Close) { // Pipeline closed, close the connection _send.Complete(); _open.Cancel(); break; } } } catch (OperationCanceledException) { break; } catch (Exception e) { ProxyEventSource.Log.StreamException(this, codec.Stream, e); break; } } }
/// <summary> /// Returns a formatted string of the name of a row, e.g. displayed in the FileTree or ProjectKeyframeOverview. /// </summary> /// <param name="detectable"></param> /// <returns></returns> public static string GetDescriptiveName(this IFragment detectable) { string name = detectable.DataFormat.GetDescriptiveName(); ICodecStream codecStream = detectable as ICodecStream; if (codecStream != null && !string.IsNullOrEmpty(codecStream.Name)) { name += string.Format(" ({0})", codecStream.Name); } if ((detectable.DataFormat == CodecID.Unknown) && (detectable.Detectors != null && detectable.Detectors.Count() > 0) && !(detectable.Detectors.First() is ICodecDetector)) { name = detectable.Detectors.First().Name; } if (detectable.IsFragmented || detectable.FragmentContainer != null) { name += string.Format(" (fragment {0})", detectable.FragmentIndex + 1); } return(name); }
/// <summary> /// Close stream when producers finish /// </summary> /// <param name="codec"></param> /// <returns></returns> protected override async Task CloseStreamAsync(ICodecStream <HybridConnectionStream> codec) { try { if (_open.IsCancellationRequested) { // User is asking for a graceful close, shutdown properly await codec.Stream.ShutdownAsync(new CancellationTokenSource( ServiceBusRelay._closeTimeout).Token).ConfigureAwait(false); } } catch { } try { // ... then gracefully close await codec.Stream.CloseAsync(new CancellationTokenSource( ServiceBusRelay._closeTimeout).Token).ConfigureAwait(false); } catch (OperationCanceledException) { } catch (Exception e) { ProxyEventSource.Log.StreamException(this, codec.Stream, e); } }
public void Add(ICodecStream codecStream) { _codecStreams.Add(codecStream); }
/// <summary> /// Close stream orderly when stream is done. Not called when parent faulted. /// </summary> /// <param name="codec"></param> /// <returns></returns> protected abstract Task CloseStreamAsync(ICodecStream <S> codec);
public CodecStream(ICodecStream iCodecStream) { _iCodecStream = iCodecStream; _keyFrames = new KeyFrameList(); }
public RescannedCodecStream(ICodecStream codecStream, IDataPacket data) { _codecStream = codecStream; _data = data; }
protected override async Task CloseStreamAsync(ICodecStream <WebSocketStream> codec) { var ct = new CancellationTokenSource(_closeTimeout).Token; await Try.Async(codec.Stream.CloseAsync, ct).ConfigureAwait(false); }
protected override async Task CloseStreamAsync(ICodecStream<WebSocketStream> codec) { try { await codec.Stream.CloseAsync(new CancellationTokenSource(_closeTimeout).Token); } catch {} }
private IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter) { return(_backgroundDataBlockScanner.GetData(codecStream, progressReporter)); }
protected override async Task SendConsumerAsync( ICodecStream <HybridConnectionStream> codec) { while (!_open.IsCancellationRequested) { try { if (_lastMessage == null) { if (_send.Completion.IsCompleted) { // Pipeline closed, close the connection _receive.Complete(); _open.Cancel(); break; } try { _lastMessage = await _send.ReceiveAsync(_timeout, _open.Token).ConfigureAwait(false); var data = _lastMessage.Content as DataMessage; if (data != null) { data.SequenceNumber = _nextSendSequenceNumber++; } } catch (TimeoutException) { _lastMessage = Message.Create(StreamId, _remoteId, PollRequest.Create((ulong)_timeout.TotalMilliseconds)); } catch (OperationCanceledException) when(!_open.IsCancellationRequested) { _lastMessage = Message.Create(StreamId, _remoteId, PollRequest.Create((ulong)_timeout.TotalMilliseconds)); } } // // Every write on the hybrid connection stream right now results in a binary // message not an individual fragment. // when using the codec directly on the stream then the first write confuses the // proxy decoder, which assumes a websocket message contains an entire message. // Override here to buffer the entire message in a memory stream before writing. // using (var mem = new MemoryStream()) { _lastMessage.Encode(mem, CodecId.Mpack); var buffered = mem.ToArray(); await codec.Stream.WriteAsync( buffered, 0, buffered.Length, _open.Token).ConfigureAwait(false); } await codec.Stream.FlushAsync(_open.Token).ConfigureAwait(false); _lastMessage.Dispose(); _lastMessage = null; } catch (OperationCanceledException) { break; } catch (Exception e) { ProxyEventSource.Log.StreamException(this, codec.Stream, e); break; } } }
public CodecStreamNode(ICodecStream stream, CodecID dataFormat, bool isH264AVCC, KeyFrameList keyFrameList, RangeList dataRanges) : this(stream.Name, dataFormat, isH264AVCC, stream.AbsoluteStartOffset, stream.Length, keyFrameList, dataRanges) { }
public bool Contains(ICodecStream codecStream) { return(_codecStreams.Contains(codecStream)); }
private static void AddUnknownCodecStream(IDataBlockBuilder dataBlockBuilder, ICodecStream completeCodecStream) { ICodecStreamBuilder relativeCodecStreamBuilder = dataBlockBuilder.AddCodecStream(); relativeCodecStreamBuilder.StreamNumber = completeCodecStream.StreamNumber; relativeCodecStreamBuilder.Name = completeCodecStream.Name; relativeCodecStreamBuilder.DataFormat = CodecID.Unknown; relativeCodecStreamBuilder.Detector = new UnknownFormatDetector(); relativeCodecStreamBuilder.Data = completeCodecStream.InputFile.CreateDataPacket().GetSubPacket(0L, completeCodecStream.Length); relativeCodecStreamBuilder.IsFragmented = false; relativeCodecStreamBuilder.AbsoluteStartOffset = completeCodecStream.AbsoluteStartOffset; }
/// <summary> /// Checks whether the <paramref name="newDataBlock"/> are not different from /// the initially detected <paramref name="initialDataBlock"/>. /// </summary> /// <param name="initialDataBlock">the initially detected data block</param> /// <param name="newDataBlock">the new data block to compare</param> /// <returns>whether the data block is correct</returns> private static bool IsDataBlockCorrect(IDataBlock initialDataBlock, IDataBlock newDataBlock) { if (newDataBlock == null) { return(false); } if (newDataBlock.InputFile != initialDataBlock.InputFile) { return(false); } if (newDataBlock.StartOffset != 0L) { return(false); } if (newDataBlock.Length != initialDataBlock.Length) { return(false); // Size of data block differs } if (newDataBlock.IsFullFile != initialDataBlock.IsFullFile) { return(false); // IsFullFile property differs } if (newDataBlock.CodecStreams == null) { return(initialDataBlock.CodecStreams.Count == 0); } // Codec streams can be divided into fragments when a maximum header count is reached // The data block object contains the fragmented codec streams, // the newDataBlock object contains the unfragmented codec streams. // This is handled by the 'Where' clause. // Also, codec streams that are split into multiple parts in the data block // are represented by a single codec stream in the 'newDataBlock' object. // This is handled by the 'GroupBy' clause. int initialCodecStreamCount = initialDataBlock.CodecStreams.Where(c => (c.FragmentIndex == 0)).GroupBy(c => c.StreamNumber).Count(); if (initialCodecStreamCount != newDataBlock.CodecStreams.Count) { return(false); // Codec stream count differs } // Check the codec and container stream detectors used foreach (ICodecStream initialCodecStream in initialDataBlock.CodecStreams) { if (initialCodecStream.FragmentIndex == 0) { int streamNumber = initialCodecStream.StreamNumber; ICodecStream newCodecStream = newDataBlock.CodecStreams.Where(c => c.StreamNumber == streamNumber).FirstOrDefault(); if (newCodecStream == null) { return(false); // Codec stream not found in rescanned data block } IDetector initialDetector = initialCodecStream.Detectors.FirstOrDefault(); if (!IsCompatibleCodec(initialDetector, newCodecStream.DataFormat)) { //It is possible that the container reports an incorrect format of the datastream //Therefore disabled: //return false; } } } return(true); }
public void TearDown() { _codecStream = null; }
public void SetUp() { _codecStream = new ContentBase(CodecStreamName, CodecStreamFormat, _dataPacket); }