public bool FragmentedStreamAlreadySaved(IFragment fragment)
{
if (fragment.FragmentContainer == null)
{
return(false); // This is not a fragmented stream
}
if (fragment is IDataBlock)
{
if (_handledFragmentContainers.Contains(fragment.FragmentContainer))
{
return(true);
}
_handledFragmentContainers.Add(fragment.FragmentContainer);
}
else if (fragment is ICodecStream)
{
ICodecStream codecStream = fragment as ICodecStream;
var pair = new Pair <IFragmentContainer, string>(codecStream.DataBlock.FragmentContainer, codecStream.Name);
if (_handledFragmentContainerStreams.Contains(pair))
{
return(true);
}
_handledFragmentContainerStreams.Add(pair);
}
return(false);
}
private int SaveItem(object item, IDataReaderPool dataReaderPool, string path, OverallProgressReporter overallProgressReporter, HandledContainers handledContainers, bool createForensicIntegrityLog)
{
IInputFile inputFile = item as IInputFile;
if (inputFile != null)
{
return(SaveInputFile(inputFile, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog));
}
IDataBlock dataBlock = item as IDataBlock;
if (dataBlock != null)
{
return(SaveDataBlock(dataBlock, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog));
}
ICodecStream codecStream = item as ICodecStream;
if (codecStream != null)
{
path = string.Format("{0}__{1}", path, ReplaceIllegalPathCharactersByUnderscore(codecStream.Name));
return(SaveCodecStream(codecStream, dataReaderPool, path, overallProgressReporter, handledContainers, createForensicIntegrityLog));
}
return(0); // TODO: is this an error?
}
private static string GetFileName(object item)
{
IInputFile inputFile = item as IInputFile;
if (inputFile != null)
{
return(Path.GetFileName(inputFile.Name));
}
IDataBlock containerStream = item as IDataBlock;
if (containerStream != null)
{
return(Path.GetFileName(containerStream.InputFile.Name));
}
ICodecStream codecStream = item as ICodecStream;
if (codecStream != null)
{
return(Path.GetFileName(codecStream.InputFile.Name));
}
Debug.Fail("item should be of type IInputFile, IDataBlock or ICodecStream");
return(string.Empty);
}
//internal IDataPacket GetData(ICodecStream codecStream)
//{
// return GetData(codecStream, new NullProgressReporter());
//}
internal IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter)
{
using (IDataReaderPool dataReaderPool = _createDataReaderPool())
{
return(_dataBlockScanner.GetData(codecStream, progressReporter, dataReaderPool));
}
}
public void CountNumberOfParts(IEnumerable <object> items)
{
int codecStreamCount = 0;
int containerStreamCount = 0;
int uniqueStreamCount = 0;
HashSet <IFragmentContainer> handledFragmentContainers = new HashSet <IFragmentContainer>();
HashSet <Pair <IFragmentContainer, string> > handledFragmentContainerStreams = new HashSet <Pair <IFragmentContainer, string> >();
foreach (object item in items)
{
IInputFile inputFile = item as IInputFile;
IDataBlock containerStream = item as IDataBlock;
ICodecStream codecStream = item as ICodecStream;
if (inputFile != null)
{
IList <IDataBlock> dataBlocks = inputFile.Project.GetDataBlocks(inputFile);
containerStreamCount += dataBlocks.Count;
foreach (IDataBlock dataBlock in dataBlocks)
{
codecStreamCount += dataBlock.CodecStreams.Count;
uniqueStreamCount += CountUniqueStreams(handledFragmentContainers, handledFragmentContainerStreams, dataBlock);
}
}
else if (containerStream != null)
{
containerStreamCount++;
codecStreamCount += containerStream.CodecStreams.Count;
uniqueStreamCount += CountUniqueStreams(handledFragmentContainers, handledFragmentContainerStreams, containerStream);
}
else if (codecStream != null)
{
// Count the number of fragments for this codecStream.
// The codec stream can be part of a fragmented codec stream
// which on its turn can be part of a fragmented container stream
if (codecStream.DataBlock.FragmentContainer != null)
{
foreach (IDataBlock containerFragment in codecStream.DataBlock.FragmentContainer)
{
codecStreamCount += CountCodecStreamFragments(containerFragment, codecStream.Name);
}
}
else
{
codecStreamCount += CountCodecStreamFragments(codecStream.DataBlock, codecStream.Name);
}
if (FragmentedStreamAlreadySaved(handledFragmentContainers, handledFragmentContainerStreams, codecStream) == false)
{
uniqueStreamCount++;
}
}
}
// Progress is reported for each:
// - codec stream (twice, once during DetectData and once during GetDataPacket)
// - container stream and codec stream that is saved
_partCount = codecStreamCount * 2 + containerStreamCount + uniqueStreamCount;
}
public FileScanResult(IFragment sourceFragment, IResultNode result,
IDataBlock resultDataBlock, ICodecStream resultCodecStream,
int availableCodecStreams)
{
IsValid = true;
_sourceFragment = sourceFragment;
_result = result;
_resultDataBlock = resultDataBlock;
_resultCodecStream = resultCodecStream;
_availableCodecStreams = availableCodecStreams;
}
/// <summary>
/// Send consumer, reading messages one by one from stream and writes
/// to websocket stream.
/// </summary>
/// <returns></returns>
protected virtual async Task SendConsumerAsync(ICodecStream <S> codec)
{
if (_lastMessage == null)
{
_lastMessage = Message.Create(StreamId, _remoteId,
PollRequest.Create((ulong)_timeout.TotalMilliseconds));
}
while (!_open.IsCancellationRequested)
{
try {
if (_lastMessage == null)
{
if (_send.Completion.IsCompleted)
{
// Pipeline closed, close the connection
_receive.Complete();
_open.Cancel();
break;
}
try {
_lastMessage = await _send.ReceiveAsync(_timeout,
_open.Token).ConfigureAwait(false);
if (_lastMessage.Content is DataMessage data)
{
data.SequenceNumber = _nextSendSequenceNumber++;
}
}
catch (TimeoutException) {
_lastMessage = Message.Create(StreamId, _remoteId,
PollRequest.Create((ulong)_timeout.TotalMilliseconds));
}
catch (OperationCanceledException) when(!_open.IsCancellationRequested)
{
_lastMessage = Message.Create(StreamId, _remoteId,
PollRequest.Create((ulong)_timeout.TotalMilliseconds));
}
}
await codec.WriteAsync(_lastMessage, _open.Token).ConfigureAwait(false);
await codec.Stream.FlushAsync(_open.Token).ConfigureAwait(false);
_lastMessage.Dispose();
_lastMessage = null;
}
catch (OperationCanceledException) {
break;
}
catch (Exception e) {
ProxyEventSource.Log.StreamException(this, codec.Stream, e);
break;
}
}
}
public void TearDown()
{
_mockRepository = null;
_codecStream = null;
_dataWriter = null;
_progressReporter1 = null;
_detectors = null;
_dataReaderPool = null;
_createDataWriter = null;
_createSubProgressReporter = null;
_saveAsSeparateFiles = null;
}
/// <summary>
/// Gets the data for the given <paramref name="codecStream"/> and
/// reports progress to the given <paramref name="progressReporter"/>.
/// This rescans the parent (container) of the codecStream to determine
/// the data of codecStream.
/// </summary>
/// <param name="codecStream">the codec stream to retrieve the data for</param>
/// <param name="progressReporter">the progress reporter</param>
/// <param name="dataReaderPool">The shared pool of file data readers</param>
/// <returns>the root node of the results</returns>
public IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter, IDataReaderPool dataReaderPool)
{
IDataPacket data;
if (_dataCache.TryGetValue(codecStream, out data))
{
return(new RescannedCodecStream(codecStream, data));
}
IDataBlock containerStreamDataBlock = codecStream.DataBlock;
using (IDataReader dataReader = new ProgressDataReader(new FragmentedDataReader(containerStreamDataBlock, dataReaderPool), progressReporter))
{
IDetector containerDetector = containerStreamDataBlock.Detectors.FirstOrDefault();
IScanContext scanContext = _createScanContext(containerStreamDataBlock.InputFile.Project);
scanContext.Detectors = new[] { containerDetector };
IDataBlockBuilder containerStreamDataBlockBuilder = _createDataBlockBuilder();
containerStreamDataBlockBuilder.Detectors = scanContext.Detectors;
containerStreamDataBlockBuilder.InputFile = containerStreamDataBlock.InputFile;
IDataBlock newContainerStream = containerDetector.DetectData(dataReader, containerStreamDataBlockBuilder, scanContext);
// Check cancellation request
if (progressReporter.CancellationPending)
{
return(null);
}
// Check results / container stream
if (newContainerStream == null)
{
RetrievedDetectableIsDifferentThanTheOneRetrievedDuringTheInitialScan();
return(null);
}
data = newContainerStream.CodecStreams.Where(c => c.StreamNumber == codecStream.StreamNumber).FirstOrDefault();
if ((data == null) || (data.Length < codecStream.EndOffset))
{
RetrievedDetectableIsDifferentThanTheOneRetrievedDuringTheInitialScan();
return(null);
}
data = data.GetSubPacket(codecStream.StartOffset, codecStream.Length);
}
CacheData(codecStream, data);
return(new RescannedCodecStream(codecStream, data));
}
public void SetUp()
{
_mockRepository = new MockRepository();
_codecStream = MockRepository.GenerateStub <ICodecStream>();
_dataWriter = _mockRepository.StrictMock <IDataWriter>();
_progressReporter1 = MockRepository.GenerateStub <IProgressReporter>();
_detectors = Enumerable.Empty <IDetector>();
_dataReaderPool = MockRepository.GenerateStub <IDataReaderPool>();
_createDataWriter = MockRepository.GenerateStub <Creator <IDataWriter, string> >();
_createSubProgressReporter = MockRepository.GenerateStub <Creator <IProgressReporter, IProgressReporter, long, long, long> >();
_forensicIntegrityLog = MockRepository.GenerateStub <IForensicIntegrityLog>();
_createDataWriter.Stub(x => x(FileName)).Return(_dataWriter).Repeat.Once();
_saveAsSeparateFiles = new SaveAsSeparateFiles(_createDataWriter, _createSubProgressReporter, TestFramework.DataBlockScanner, _forensicIntegrityLog);
}
private IList <ICodecStream> BuildCodecStreams(IDataBlock dataBlock)
{
List <ICodecStream> codecStreams = new List <ICodecStream>();
IFragment previousFragment = null;
foreach (ICodecStreamBuilder codecStreamBuilder in _codecStreamBuilders)
{
codecStreamBuilder.DataBlock = dataBlock;
codecStreamBuilder.PreviousFragment = previousFragment;
ICodecStream codecStream = codecStreamBuilder.Build();
codecStreams.Add(codecStream);
previousFragment = codecStream;
}
return(codecStreams.AsReadOnly());
}
internal CompleteCodecStreamScanner(IDataScanner dataScanner, IDataBlockBuilder dataBlockBuilder, ICodecStream completeCodecStream, IProgressReporter progressReporter)
{
_dataScanner = dataScanner;
_dataBlockBuilder = dataBlockBuilder;
_completeCodecStream = completeCodecStream;
_progressReporter = progressReporter;
_codecStreamFragments = new List <IFragment>();
long length = _completeCodecStream.Length;
_maxUnknownDataForCompleteStream = Math.Max((MaxPercentageUnkownDataForCompleteStream * length) / 100, Math.Min((2 * MaxUnparsableBytes), length));
long detectBytes = Math.Min(length, MaxBytesForDetect);
_maxUnknownDataForDetect = Math.Max((MaxPercentageUnkownDataForDetect * detectBytes) / 100, Math.Min((2 * MaxUnparsableBytes), detectBytes));
}
/// <summary>
/// Wrapper that calls dispose on the stream - at a minimum...
/// </summary>
/// <param name="codec"></param>
/// <param name="parentHasFaulted"></param>
/// <returns></returns>
private async Task CloseStreamAsync(ICodecStream <S> codec,
bool parentHasFaulted)
{
ProxyEventSource.Log.StreamClosing(this, codec.Stream);
if (!parentHasFaulted)
{
await CloseStreamAsync(codec).ConfigureAwait(false);
}
try {
codec.Stream.Dispose();
}
catch (Exception e) {
ProxyEventSource.Log.HandledExceptionAsInformation(codec.Stream, e);
}
ProxyEventSource.Log.StreamClosed(this, codec.Stream);
}
/// <summary>
/// Receive producer, reading messages one by one from relay and
/// writing to receive block. Manages stream as well.
/// </summary>
/// <returns></returns>
protected virtual async Task ReceiveProducerAsync(ICodecStream <S> codec)
{
while (!_open.IsCancellationRequested)
{
try {
// Read message and send to source block
var message = await codec.ReadAsync <Message>(_open.Token).ConfigureAwait(false);
if (message != null)
{
if (message.Content is DataMessage data && data.SequenceNumber != _nextReceiveSequenceNumber++)
{
// TODO: Implement poll for previous message
if (data.SequenceNumber > _nextReceiveSequenceNumber - 1)
{
ProxyEventSource.Log.MissingData(this, data, _nextReceiveSequenceNumber - 1);
message.Error = (int)SocketError.Missing;
}
else
{
ProxyEventSource.Log.DuplicateData(this, data, _nextReceiveSequenceNumber - 1);
// Do not even bother to forward
continue;
}
}
if (!await _receive.SendAsync(message, _open.Token).ConfigureAwait(false) ||
message.TypeId == MessageContent.Close)
{
// Pipeline closed, close the connection
_send.Complete();
_open.Cancel();
break;
}
}
}
catch (OperationCanceledException) {
break;
}
catch (Exception e) {
ProxyEventSource.Log.StreamException(this, codec.Stream, e);
break;
}
}
}
/// <summary>
/// Returns a formatted string of the name of a row, e.g. displayed in the FileTree or ProjectKeyframeOverview.
/// </summary>
/// <param name="detectable"></param>
/// <returns></returns>
public static string GetDescriptiveName(this IFragment detectable)
{
string name = detectable.DataFormat.GetDescriptiveName();
ICodecStream codecStream = detectable as ICodecStream;
if (codecStream != null && !string.IsNullOrEmpty(codecStream.Name))
{
name += string.Format(" ({0})", codecStream.Name);
}
if ((detectable.DataFormat == CodecID.Unknown) && (detectable.Detectors != null && detectable.Detectors.Count() > 0) && !(detectable.Detectors.First() is ICodecDetector))
{
name = detectable.Detectors.First().Name;
}
if (detectable.IsFragmented || detectable.FragmentContainer != null)
{
name += string.Format(" (fragment {0})", detectable.FragmentIndex + 1);
}
return(name);
}
/// <summary>
/// Close stream when producers finish
/// </summary>
/// <param name="codec"></param>
/// <returns></returns>
protected override async Task CloseStreamAsync(ICodecStream <HybridConnectionStream> codec)
{
try {
if (_open.IsCancellationRequested)
{
// User is asking for a graceful close, shutdown properly
await codec.Stream.ShutdownAsync(new CancellationTokenSource(
ServiceBusRelay._closeTimeout).Token).ConfigureAwait(false);
}
}
catch { }
try {
// ... then gracefully close
await codec.Stream.CloseAsync(new CancellationTokenSource(
ServiceBusRelay._closeTimeout).Token).ConfigureAwait(false);
}
catch (OperationCanceledException) {
}
catch (Exception e) {
ProxyEventSource.Log.StreamException(this, codec.Stream, e);
}
}
public void Add(ICodecStream codecStream)
{
_codecStreams.Add(codecStream);
}
/// <summary> /// Close stream orderly when stream is done. Not called when parent faulted. /// </summary> /// <param name="codec"></param> /// <returns></returns> protected abstract Task CloseStreamAsync(ICodecStream <S> codec);
public CodecStream(ICodecStream iCodecStream)
{
_iCodecStream = iCodecStream;
_keyFrames = new KeyFrameList();
}
public RescannedCodecStream(ICodecStream codecStream, IDataPacket data)
{
_codecStream = codecStream;
_data = data;
}
protected override async Task CloseStreamAsync(ICodecStream <WebSocketStream> codec)
{
var ct = new CancellationTokenSource(_closeTimeout).Token;
await Try.Async(codec.Stream.CloseAsync, ct).ConfigureAwait(false);
}
protected override async Task CloseStreamAsync(ICodecStream<WebSocketStream> codec) {
try {
await codec.Stream.CloseAsync(new CancellationTokenSource(_closeTimeout).Token);
}
catch {}
}
private IDataPacket GetData(ICodecStream codecStream, IProgressReporter progressReporter)
{
return(_backgroundDataBlockScanner.GetData(codecStream, progressReporter));
}
protected override async Task SendConsumerAsync(
ICodecStream <HybridConnectionStream> codec)
{
while (!_open.IsCancellationRequested)
{
try {
if (_lastMessage == null)
{
if (_send.Completion.IsCompleted)
{
// Pipeline closed, close the connection
_receive.Complete();
_open.Cancel();
break;
}
try {
_lastMessage = await _send.ReceiveAsync(_timeout,
_open.Token).ConfigureAwait(false);
var data = _lastMessage.Content as DataMessage;
if (data != null)
{
data.SequenceNumber = _nextSendSequenceNumber++;
}
}
catch (TimeoutException) {
_lastMessage = Message.Create(StreamId, _remoteId,
PollRequest.Create((ulong)_timeout.TotalMilliseconds));
}
catch (OperationCanceledException) when(!_open.IsCancellationRequested)
{
_lastMessage = Message.Create(StreamId, _remoteId,
PollRequest.Create((ulong)_timeout.TotalMilliseconds));
}
}
//
// Every write on the hybrid connection stream right now results in a binary
// message not an individual fragment.
// when using the codec directly on the stream then the first write confuses the
// proxy decoder, which assumes a websocket message contains an entire message.
// Override here to buffer the entire message in a memory stream before writing.
//
using (var mem = new MemoryStream()) {
_lastMessage.Encode(mem, CodecId.Mpack);
var buffered = mem.ToArray();
await codec.Stream.WriteAsync(
buffered, 0, buffered.Length, _open.Token).ConfigureAwait(false);
}
await codec.Stream.FlushAsync(_open.Token).ConfigureAwait(false);
_lastMessage.Dispose();
_lastMessage = null;
}
catch (OperationCanceledException) {
break;
}
catch (Exception e) {
ProxyEventSource.Log.StreamException(this, codec.Stream, e);
break;
}
}
}
public CodecStreamNode(ICodecStream stream, CodecID dataFormat, bool isH264AVCC, KeyFrameList keyFrameList, RangeList dataRanges) :
this(stream.Name, dataFormat, isH264AVCC, stream.AbsoluteStartOffset, stream.Length, keyFrameList, dataRanges)
{
}
public bool Contains(ICodecStream codecStream)
{
return(_codecStreams.Contains(codecStream));
}
private static void AddUnknownCodecStream(IDataBlockBuilder dataBlockBuilder, ICodecStream completeCodecStream)
{
ICodecStreamBuilder relativeCodecStreamBuilder = dataBlockBuilder.AddCodecStream();
relativeCodecStreamBuilder.StreamNumber = completeCodecStream.StreamNumber;
relativeCodecStreamBuilder.Name = completeCodecStream.Name;
relativeCodecStreamBuilder.DataFormat = CodecID.Unknown;
relativeCodecStreamBuilder.Detector = new UnknownFormatDetector();
relativeCodecStreamBuilder.Data = completeCodecStream.InputFile.CreateDataPacket().GetSubPacket(0L, completeCodecStream.Length);
relativeCodecStreamBuilder.IsFragmented = false;
relativeCodecStreamBuilder.AbsoluteStartOffset = completeCodecStream.AbsoluteStartOffset;
}
/// <summary>
/// Checks whether the <paramref name="newDataBlock"/> are not different from
/// the initially detected <paramref name="initialDataBlock"/>.
/// </summary>
/// <param name="initialDataBlock">the initially detected data block</param>
/// <param name="newDataBlock">the new data block to compare</param>
/// <returns>whether the data block is correct</returns>
private static bool IsDataBlockCorrect(IDataBlock initialDataBlock, IDataBlock newDataBlock)
{
if (newDataBlock == null)
{
return(false);
}
if (newDataBlock.InputFile != initialDataBlock.InputFile)
{
return(false);
}
if (newDataBlock.StartOffset != 0L)
{
return(false);
}
if (newDataBlock.Length != initialDataBlock.Length)
{
return(false); // Size of data block differs
}
if (newDataBlock.IsFullFile != initialDataBlock.IsFullFile)
{
return(false); // IsFullFile property differs
}
if (newDataBlock.CodecStreams == null)
{
return(initialDataBlock.CodecStreams.Count == 0);
}
// Codec streams can be divided into fragments when a maximum header count is reached
// The data block object contains the fragmented codec streams,
// the newDataBlock object contains the unfragmented codec streams.
// This is handled by the 'Where' clause.
// Also, codec streams that are split into multiple parts in the data block
// are represented by a single codec stream in the 'newDataBlock' object.
// This is handled by the 'GroupBy' clause.
int initialCodecStreamCount = initialDataBlock.CodecStreams.Where(c => (c.FragmentIndex == 0)).GroupBy(c => c.StreamNumber).Count();
if (initialCodecStreamCount != newDataBlock.CodecStreams.Count)
{
return(false); // Codec stream count differs
}
// Check the codec and container stream detectors used
foreach (ICodecStream initialCodecStream in initialDataBlock.CodecStreams)
{
if (initialCodecStream.FragmentIndex == 0)
{
int streamNumber = initialCodecStream.StreamNumber;
ICodecStream newCodecStream = newDataBlock.CodecStreams.Where(c => c.StreamNumber == streamNumber).FirstOrDefault();
if (newCodecStream == null)
{
return(false); // Codec stream not found in rescanned data block
}
IDetector initialDetector = initialCodecStream.Detectors.FirstOrDefault();
if (!IsCompatibleCodec(initialDetector, newCodecStream.DataFormat))
{
//It is possible that the container reports an incorrect format of the datastream
//Therefore disabled:
//return false;
}
}
}
return(true);
}
public void TearDown()
{
_codecStream = null;
}
public void SetUp()
{
_codecStream = new ContentBase(CodecStreamName, CodecStreamFormat, _dataPacket);
}
