internal void CompleteEncoderSession(EncoderSession session, int section, ArchiveDecoderSection definition, EncoderStorage[] storageList) { if (!mEncoderSessions.Remove(session)) { throw new InternalFailureException(); } System.Diagnostics.Debug.Assert(mDecoderSections[section] == null); mDecoderSections[section] = definition; // TODO: we can write storage lazily (just remember the streams in a list) and don't have to block the caller foreach (var storage in storageList) { var stream = storage.GetFinalStream(); var offset = mAppendPosition; var length = stream.Length; mAppendPosition = checked (offset + length); var checksum = storage.GetFinalChecksum(); mFileSections.Add(new ArchiveFileSection(offset, length, checksum)); mArchiveStream.Position = offset; stream.Position = 0; stream.CopyTo(mArchiveStream); } }
private static void SelectStream( ArchiveMetadata archiveMetadata, ArchiveDecoderSection sectionMetadata, DecoderInputMetadata selector, ReaderNode[] streams, DecoderNode[] decoders, out ReaderNode result, out long length) { var decoderIndex = selector.DecoderIndex; if (decoderIndex.HasValue) { length = sectionMetadata.Decoders[decoderIndex.Value].OutputStreams[selector.StreamIndex].Length; result = decoders[decoderIndex.Value].GetOutputStream(selector.StreamIndex); } else { length = archiveMetadata.FileSections[selector.StreamIndex].Length; result = streams[selector.StreamIndex]; } }
internal void CompleteEncoderSession(EncoderSession session, int section, ArchiveDecoderSection definition, EncoderStorage[] storageList) { if (!mEncoderSessions.Remove(session)) throw new InternalFailureException(); System.Diagnostics.Debug.Assert(mDecoderSections[section] == null); mDecoderSections[section] = definition; // TODO: we can write storage lazily (just remember the streams in a list) and don't have to block the caller foreach (var storage in storageList) { var stream = storage.GetFinalStream(); var offset = mAppendPosition; var length = stream.Length; mAppendPosition = checked(offset + length); var checksum = storage.GetFinalChecksum(); mFileSections.Add(new ArchiveFileSection(offset, length, checksum)); mArchiveStream.Position = offset; stream.Position = 0; stream.CopyTo(mArchiveStream); } }
private void WriteDecoderSection(ArchiveDecoderSection definition, ref int firstStreamIndex) { WriteNumber(definition.Decoders.Length); var inputOffset = new int[definition.Decoders.Length]; var outputOffset = new int[definition.Decoders.Length]; for (int i = 1; i < definition.Decoders.Length; i++) { inputOffset[i] = inputOffset[i - 1] + definition.Decoders[i - 1].InputStreams.Length; outputOffset[i] = outputOffset[i - 1] + definition.Decoders[i - 1].OutputStreams.Length; } for (int i = 0; i < definition.Decoders.Length; i++) { var decoder = definition.Decoders[i]; var id = decoder.DecoderType.Encode(); var multiStream = decoder.InputStreams.Length != 1 || decoder.OutputStreams.Length != 1; var settings = decoder.Settings; var hasSettings = !settings.IsDefaultOrEmpty; System.Diagnostics.Debug.Assert(!id.IsDefaultOrEmpty && id.Length <= 15); var flags = (byte)id.Length; if (multiStream) flags |= 0x10; if (hasSettings) flags |= 0x20; WriteByte(flags); foreach (var bt in id) WriteByte(bt); if (multiStream) { WriteNumber(decoder.InputStreams.Length); WriteNumber(decoder.OutputStreams.Length); } if (hasSettings) { WriteNumber(settings.Length); foreach (var bt in settings) WriteByte(bt); } } for (int i = 0; i < definition.Decoders.Length; i++) { var decoder = definition.Decoders[i]; for (int j = 0; j < decoder.InputStreams.Length; j++) { var input = decoder.InputStreams[j]; if (input.DecoderIndex.HasValue) { WriteNumber(inputOffset[i] + j); WriteNumber(outputOffset[input.DecoderIndex.Value] + input.StreamIndex); } } } int fileStreamSections = 0; foreach (var decoder in definition.Decoders) foreach (var input in decoder.InputStreams) if (!input.DecoderIndex.HasValue) fileStreamSections += 1; if (fileStreamSections > 1) foreach (var decoder in definition.Decoders) foreach (var input in decoder.InputStreams) if (!input.DecoderIndex.HasValue) WriteNumber(input.StreamIndex - firstStreamIndex); firstStreamIndex += fileStreamSections; }
public ArchiveFileModel ReadMetadata(Stream stream, PasswordStorage password) { if (mIsRunning) { throw new InvalidOperationException("Recursive invocation."); } try { mIsRunning = true; // reset fields to detect bugs mFileNames = null; mItemsWithoutStreamMarkers = null; mEmptyFileMarkers = null; mDeletionMarkers = null; mOffsets = null; mAttributes = null; mCDates = null; mMDates = null; mADates = null; mItemCount = -1; mItemsWithoutStream = -1; var metadata = ReadMetadataCore(stream, password); mRootFolder = new ArchivedFolder.Builder(); mItemMap = new Dictionary <ArchivedFolder.Builder, List <ArchivedItem.Builder> >(); mItemMap.Add(mRootFolder, new List <ArchivedItem.Builder>()); mFiles = new HashSet <ArchivedFile.Builder>(); mStreamMap = new List <ArchivedFile.Builder>(); mSectionMap = ImmutableArray.CreateBuilder <int>(metadata.DecoderSections.Length + 1); ArchiveDecoderSection currentDecoder = null; int currentSectionIndex = -1; int currentStreamIndex = 0; int currentStreamCount = 0; int currentEmptyIndex = 0; for (int currentFileIndex = 0; currentFileIndex < mFileNames.Count; currentFileIndex++) { var filename = mFileNames[currentFileIndex]; bool hasStream = false; ArchivedFile.Builder file = null; if (mItemsWithoutStreamMarkers != null && mItemsWithoutStreamMarkers[currentFileIndex]) { var isFile = (mEmptyFileMarkers == null || mEmptyFileMarkers[currentEmptyIndex]); var isDeletionMarker = (mDeletionMarkers != null && mDeletionMarkers[currentEmptyIndex]); if (isDeletionMarker) { RemoveItem(mItemMap[mRootFolder], filename, 0); } else { file = AddItem(mItemMap[mRootFolder], filename, 0, isFile); } currentEmptyIndex++; } else { hasStream = true; file = AddItem(mItemMap[mRootFolder], filename, 0, true); } if (file != null) { if (mOffsets != null) { file.Offset = mOffsets[currentFileIndex] ?? 0; } if (mAttributes != null) { var attr = mAttributes[currentFileIndex]; if (attr.HasValue && (attr.Value & ArchivedAttributesExtensions.DirectoryAttribute) != 0) { continue; } file.Attributes = attr; } if (mCDates != null) { file.Creation = mCDates[currentFileIndex]; } if (mMDates != null) { file.LastWrite = mMDates[currentFileIndex]; } if (mADates != null) { file.LastAccess = mADates[currentFileIndex]; } if (hasStream) { while (currentStreamIndex == currentStreamCount) { if (currentSectionIndex == metadata.DecoderSections.Length - 1) { throw new InvalidDataException(); } currentDecoder = metadata.DecoderSections[++currentSectionIndex]; currentStreamCount = currentDecoder.Streams.Length; currentStreamIndex = 0; mSectionMap.Add(mStreamMap.Count); } file.Stream = new DecodedStreamIndex(currentSectionIndex, currentStreamIndex); var streamMetadata = currentDecoder.Streams[currentStreamIndex++]; file.Length = streamMetadata.Length; file.Checksum = streamMetadata.Checksum; } } if (hasStream) { mStreamMap.Add(file); } } var finalStreamMap = ImmutableList.CreateBuilder <ArchivedFile>(); foreach (var file in mStreamMap) { finalStreamMap.Add(mFiles.Contains(file) ? file.ToImmutable() : null); } if (currentStreamIndex != currentStreamCount || currentSectionIndex != metadata.DecoderSections.Length - 1) { throw new InvalidDataException(); } mSectionMap.Add(finalStreamMap.Count); return(new ArchiveFileModel(metadata, BuildFolder(mRootFolder), mSectionMap.MoveToImmutable(), finalStreamMap.ToImmutable())); } finally { mIsRunning = false; } }
public async Task Complete() { lock (mLockObject) { if (mComplete) { throw new InvalidOperationException(); } mComplete = true; Monitor.PulseAll(mLockObject); while (!mCompleteAck) { Monitor.Wait(mLockObject); } } #if NET_4_6_2 // This will probably deadlock up to .NET 4.6.1 and might be fixed in .NET 4.6.2 // The cause is that Task.WhenAll does not respect TaskCreationOptions.RunContinuationsAsynchronously await Task.WhenAll(mStorage.Select(x => x.GetCompletionTask())).ConfigureAwait(false); #else foreach (var stream in mStorage) { await stream.GetCompletionTask().ConfigureAwait(false); } #endif int totalInputCount = 0; var firstInputOffset = new int[mEncoders.Length]; for (int i = 0; i < mEncoders.Length; i++) { firstInputOffset[i] = totalInputCount; totalInputCount += mDefinition.GetEncoder(i).InputCount; } var decoders = ImmutableArray.CreateBuilder <DecoderMetadata>(mEncoders.Length); for (int i = 0; i < mEncoders.Length; i++) { var encoder = mDefinition.GetEncoder(i); var settings = encoder.Settings; var decoderType = settings.GetDecoderType(); var decoderInputCount = decoderType.GetInputCount(); var decoderInputs = ImmutableArray.CreateBuilder <DecoderInputMetadata>(decoderInputCount); for (int j = 0; j < decoderInputCount; j++) { var encoderOutput = encoder.GetOutput(j).Target; if (encoderOutput.IsStorage) { decoderInputs.Add(new DecoderInputMetadata(null, encoderOutput.Index)); } else { decoderInputs.Add(new DecoderInputMetadata(encoderOutput.Node.Index, encoderOutput.Index)); } } var decoderOutputCount = decoderType.GetOutputCount(); var decoderOutputs = ImmutableArray.CreateBuilder <DecoderOutputMetadata>(decoderOutputCount); for (int j = 0; j < decoderOutputCount; j++) { var encoderInput = encoder.GetInput(j).Source; if (encoderInput.IsContent) { System.Diagnostics.Debug.Assert(mConnections[firstInputOffset[i] + j] == null); decoderOutputs.Add(new DecoderOutputMetadata(mInput.GetFinalLength())); } else { decoderOutputs.Add(new DecoderOutputMetadata(mConnections[firstInputOffset[i] + j].GetFinalLength())); } } decoders.Add(new DecoderMetadata(decoderType, settings.SerializeSettings(), decoderInputs.MoveToImmutable(), decoderOutputs.MoveToImmutable())); } var inputSource = mDefinition.GetContentSource().Target; var definition = new ArchiveDecoderSection( decoders.MoveToImmutable(), new DecoderInputMetadata(inputSource.Node.Index, inputSource.Index), mInput.GetFinalLength(), mInput.GetFinalChecksum(), mContent.ToImmutable()); mWriter.CompleteEncoderSession(this, mSection, definition, mStorage); }
/// <summary> /// Creates a new iteration to unpack a sequence of streams from a decoded archive section. /// </summary> /// <param name="stream">A readable and seekable stream for the archive.</param> /// <param name="metadata">The metadata for the archive.</param> /// <param name="index">The decoder section index from the metadata which should be unpacked.</param> /// <param name="password">An optional password to unpack the archive content.</param> public DecodedSectionReader(Stream stream, ArchiveMetadata metadata, int index, PasswordStorage password) { mDecodedStream = new ArchiveSectionDecoder(stream, metadata, index, password); mMetadata = metadata; mDecoderSection = metadata.DecoderSections[index]; }
public async Task Complete() { lock (mLockObject) { if (mComplete) throw new InvalidOperationException(); mComplete = true; Monitor.PulseAll(mLockObject); while (!mCompleteAck) Monitor.Wait(mLockObject); } #if NET_4_6_2 // This will probably deadlock up to .NET 4.6.1 and might be fixed in .NET 4.6.2 // The cause is that Task.WhenAll does not respect TaskCreationOptions.RunContinuationsAsynchronously await Task.WhenAll(mStorage.Select(x => x.GetCompletionTask())).ConfigureAwait(false); #else foreach (var stream in mStorage) await stream.GetCompletionTask().ConfigureAwait(false); #endif int totalInputCount = 0; var firstInputOffset = new int[mEncoders.Length]; for (int i = 0; i < mEncoders.Length; i++) { firstInputOffset[i] = totalInputCount; totalInputCount += mDefinition.GetEncoder(i).InputCount; } var decoders = ImmutableArray.CreateBuilder<DecoderMetadata>(mEncoders.Length); for (int i = 0; i < mEncoders.Length; i++) { var encoder = mDefinition.GetEncoder(i); var settings = encoder.Settings; var decoderType = settings.GetDecoderType(); var decoderInputCount = decoderType.GetInputCount(); var decoderInputs = ImmutableArray.CreateBuilder<DecoderInputMetadata>(decoderInputCount); for (int j = 0; j < decoderInputCount; j++) { var encoderOutput = encoder.GetOutput(j).Target; if (encoderOutput.IsStorage) decoderInputs.Add(new DecoderInputMetadata(null, encoderOutput.Index)); else decoderInputs.Add(new DecoderInputMetadata(encoderOutput.Node.Index, encoderOutput.Index)); } var decoderOutputCount = decoderType.GetOutputCount(); var decoderOutputs = ImmutableArray.CreateBuilder<DecoderOutputMetadata>(decoderOutputCount); for (int j = 0; j < decoderOutputCount; j++) { var encoderInput = encoder.GetInput(j).Source; if (encoderInput.IsContent) { System.Diagnostics.Debug.Assert(mConnections[firstInputOffset[i] + j] == null); decoderOutputs.Add(new DecoderOutputMetadata(mInput.GetFinalLength())); } else { decoderOutputs.Add(new DecoderOutputMetadata(mConnections[firstInputOffset[i] + j].GetFinalLength())); } } decoders.Add(new DecoderMetadata(decoderType, settings.SerializeSettings(), decoderInputs.MoveToImmutable(), decoderOutputs.MoveToImmutable())); } var inputSource = mDefinition.GetContentSource().Target; var definition = new ArchiveDecoderSection( decoders.MoveToImmutable(), new DecoderInputMetadata(inputSource.Node.Index, inputSource.Index), mInput.GetFinalLength(), mInput.GetFinalChecksum(), mContent.ToImmutable()); mWriter.CompleteEncoderSession(this, mSection, definition, mStorage); }
private void WriteDecoderSection(ArchiveDecoderSection definition, ref int firstStreamIndex) { WriteNumber(definition.Decoders.Length); var inputOffset = new int[definition.Decoders.Length]; var outputOffset = new int[definition.Decoders.Length]; for (int i = 1; i < definition.Decoders.Length; i++) { inputOffset[i] = inputOffset[i - 1] + definition.Decoders[i - 1].InputStreams.Length; outputOffset[i] = outputOffset[i - 1] + definition.Decoders[i - 1].OutputStreams.Length; } for (int i = 0; i < definition.Decoders.Length; i++) { var decoder = definition.Decoders[i]; var id = decoder.DecoderType.Encode(); var multiStream = decoder.InputStreams.Length != 1 || decoder.OutputStreams.Length != 1; var settings = decoder.Settings; var hasSettings = !settings.IsDefaultOrEmpty; System.Diagnostics.Debug.Assert(!id.IsDefaultOrEmpty && id.Length <= 15); var flags = (byte)id.Length; if (multiStream) { flags |= 0x10; } if (hasSettings) { flags |= 0x20; } WriteByte(flags); foreach (var bt in id) { WriteByte(bt); } if (multiStream) { WriteNumber(decoder.InputStreams.Length); WriteNumber(decoder.OutputStreams.Length); } if (hasSettings) { WriteNumber(settings.Length); foreach (var bt in settings) { WriteByte(bt); } } } for (int i = 0; i < definition.Decoders.Length; i++) { var decoder = definition.Decoders[i]; for (int j = 0; j < decoder.InputStreams.Length; j++) { var input = decoder.InputStreams[j]; if (input.DecoderIndex.HasValue) { WriteNumber(inputOffset[i] + j); WriteNumber(outputOffset[input.DecoderIndex.Value] + input.StreamIndex); } } } int fileStreamSections = 0; foreach (var decoder in definition.Decoders) { foreach (var input in decoder.InputStreams) { if (!input.DecoderIndex.HasValue) { fileStreamSections += 1; } } } if (fileStreamSections > 1) { foreach (var decoder in definition.Decoders) { foreach (var input in decoder.InputStreams) { if (!input.DecoderIndex.HasValue) { WriteNumber(input.StreamIndex - firstStreamIndex); } } } } firstStreamIndex += fileStreamSections; }