internal void CompleteEncoderSession(EncoderSession session, int section, ArchiveDecoderSection definition, EncoderStorage[] storageList)
{
if (!mEncoderSessions.Remove(session))
{
throw new InternalFailureException();
}
System.Diagnostics.Debug.Assert(mDecoderSections[section] == null);
mDecoderSections[section] = definition;
// TODO: we can write storage lazily (just remember the streams in a list) and don't have to block the caller
foreach (var storage in storageList)
{
var stream = storage.GetFinalStream();
var offset = mAppendPosition;
var length = stream.Length;
mAppendPosition = checked (offset + length);
var checksum = storage.GetFinalChecksum();
mFileSections.Add(new ArchiveFileSection(offset, length, checksum));
mArchiveStream.Position = offset;
stream.Position = 0;
stream.CopyTo(mArchiveStream);
}
}
private static void SelectStream(
ArchiveMetadata archiveMetadata,
ArchiveDecoderSection sectionMetadata,
DecoderInputMetadata selector,
ReaderNode[] streams,
DecoderNode[] decoders,
out ReaderNode result,
out long length)
{
var decoderIndex = selector.DecoderIndex;
if (decoderIndex.HasValue)
{
length = sectionMetadata.Decoders[decoderIndex.Value].OutputStreams[selector.StreamIndex].Length;
result = decoders[decoderIndex.Value].GetOutputStream(selector.StreamIndex);
}
else
{
length = archiveMetadata.FileSections[selector.StreamIndex].Length;
result = streams[selector.StreamIndex];
}
}
internal void CompleteEncoderSession(EncoderSession session, int section, ArchiveDecoderSection definition, EncoderStorage[] storageList)
{
if (!mEncoderSessions.Remove(session))
throw new InternalFailureException();
System.Diagnostics.Debug.Assert(mDecoderSections[section] == null);
mDecoderSections[section] = definition;
// TODO: we can write storage lazily (just remember the streams in a list) and don't have to block the caller
foreach (var storage in storageList)
{
var stream = storage.GetFinalStream();
var offset = mAppendPosition;
var length = stream.Length;
mAppendPosition = checked(offset + length);
var checksum = storage.GetFinalChecksum();
mFileSections.Add(new ArchiveFileSection(offset, length, checksum));
mArchiveStream.Position = offset;
stream.Position = 0;
stream.CopyTo(mArchiveStream);
}
}
private void WriteDecoderSection(ArchiveDecoderSection definition, ref int firstStreamIndex)
{
WriteNumber(definition.Decoders.Length);
var inputOffset = new int[definition.Decoders.Length];
var outputOffset = new int[definition.Decoders.Length];
for (int i = 1; i < definition.Decoders.Length; i++)
{
inputOffset[i] = inputOffset[i - 1] + definition.Decoders[i - 1].InputStreams.Length;
outputOffset[i] = outputOffset[i - 1] + definition.Decoders[i - 1].OutputStreams.Length;
}
for (int i = 0; i < definition.Decoders.Length; i++)
{
var decoder = definition.Decoders[i];
var id = decoder.DecoderType.Encode();
var multiStream = decoder.InputStreams.Length != 1 || decoder.OutputStreams.Length != 1;
var settings = decoder.Settings;
var hasSettings = !settings.IsDefaultOrEmpty;
System.Diagnostics.Debug.Assert(!id.IsDefaultOrEmpty && id.Length <= 15);
var flags = (byte)id.Length;
if (multiStream) flags |= 0x10;
if (hasSettings) flags |= 0x20;
WriteByte(flags);
foreach (var bt in id)
WriteByte(bt);
if (multiStream)
{
WriteNumber(decoder.InputStreams.Length);
WriteNumber(decoder.OutputStreams.Length);
}
if (hasSettings)
{
WriteNumber(settings.Length);
foreach (var bt in settings)
WriteByte(bt);
}
}
for (int i = 0; i < definition.Decoders.Length; i++)
{
var decoder = definition.Decoders[i];
for (int j = 0; j < decoder.InputStreams.Length; j++)
{
var input = decoder.InputStreams[j];
if (input.DecoderIndex.HasValue)
{
WriteNumber(inputOffset[i] + j);
WriteNumber(outputOffset[input.DecoderIndex.Value] + input.StreamIndex);
}
}
}
int fileStreamSections = 0;
foreach (var decoder in definition.Decoders)
foreach (var input in decoder.InputStreams)
if (!input.DecoderIndex.HasValue)
fileStreamSections += 1;
if (fileStreamSections > 1)
foreach (var decoder in definition.Decoders)
foreach (var input in decoder.InputStreams)
if (!input.DecoderIndex.HasValue)
WriteNumber(input.StreamIndex - firstStreamIndex);
firstStreamIndex += fileStreamSections;
}
public ArchiveFileModel ReadMetadata(Stream stream, PasswordStorage password)
{
if (mIsRunning)
{
throw new InvalidOperationException("Recursive invocation.");
}
try
{
mIsRunning = true;
// reset fields to detect bugs
mFileNames = null;
mItemsWithoutStreamMarkers = null;
mEmptyFileMarkers = null;
mDeletionMarkers = null;
mOffsets = null;
mAttributes = null;
mCDates = null;
mMDates = null;
mADates = null;
mItemCount = -1;
mItemsWithoutStream = -1;
var metadata = ReadMetadataCore(stream, password);
mRootFolder = new ArchivedFolder.Builder();
mItemMap = new Dictionary <ArchivedFolder.Builder, List <ArchivedItem.Builder> >();
mItemMap.Add(mRootFolder, new List <ArchivedItem.Builder>());
mFiles = new HashSet <ArchivedFile.Builder>();
mStreamMap = new List <ArchivedFile.Builder>();
mSectionMap = ImmutableArray.CreateBuilder <int>(metadata.DecoderSections.Length + 1);
ArchiveDecoderSection currentDecoder = null;
int currentSectionIndex = -1;
int currentStreamIndex = 0;
int currentStreamCount = 0;
int currentEmptyIndex = 0;
for (int currentFileIndex = 0; currentFileIndex < mFileNames.Count; currentFileIndex++)
{
var filename = mFileNames[currentFileIndex];
bool hasStream = false;
ArchivedFile.Builder file = null;
if (mItemsWithoutStreamMarkers != null && mItemsWithoutStreamMarkers[currentFileIndex])
{
var isFile = (mEmptyFileMarkers == null || mEmptyFileMarkers[currentEmptyIndex]);
var isDeletionMarker = (mDeletionMarkers != null && mDeletionMarkers[currentEmptyIndex]);
if (isDeletionMarker)
{
RemoveItem(mItemMap[mRootFolder], filename, 0);
}
else
{
file = AddItem(mItemMap[mRootFolder], filename, 0, isFile);
}
currentEmptyIndex++;
}
else
{
hasStream = true;
file = AddItem(mItemMap[mRootFolder], filename, 0, true);
}
if (file != null)
{
if (mOffsets != null)
{
file.Offset = mOffsets[currentFileIndex] ?? 0;
}
if (mAttributes != null)
{
var attr = mAttributes[currentFileIndex];
if (attr.HasValue && (attr.Value & ArchivedAttributesExtensions.DirectoryAttribute) != 0)
{
continue;
}
file.Attributes = attr;
}
if (mCDates != null)
{
file.Creation = mCDates[currentFileIndex];
}
if (mMDates != null)
{
file.LastWrite = mMDates[currentFileIndex];
}
if (mADates != null)
{
file.LastAccess = mADates[currentFileIndex];
}
if (hasStream)
{
while (currentStreamIndex == currentStreamCount)
{
if (currentSectionIndex == metadata.DecoderSections.Length - 1)
{
throw new InvalidDataException();
}
currentDecoder = metadata.DecoderSections[++currentSectionIndex];
currentStreamCount = currentDecoder.Streams.Length;
currentStreamIndex = 0;
mSectionMap.Add(mStreamMap.Count);
}
file.Stream = new DecodedStreamIndex(currentSectionIndex, currentStreamIndex);
var streamMetadata = currentDecoder.Streams[currentStreamIndex++];
file.Length = streamMetadata.Length;
file.Checksum = streamMetadata.Checksum;
}
}
if (hasStream)
{
mStreamMap.Add(file);
}
}
var finalStreamMap = ImmutableList.CreateBuilder <ArchivedFile>();
foreach (var file in mStreamMap)
{
finalStreamMap.Add(mFiles.Contains(file) ? file.ToImmutable() : null);
}
if (currentStreamIndex != currentStreamCount || currentSectionIndex != metadata.DecoderSections.Length - 1)
{
throw new InvalidDataException();
}
mSectionMap.Add(finalStreamMap.Count);
return(new ArchiveFileModel(metadata, BuildFolder(mRootFolder), mSectionMap.MoveToImmutable(), finalStreamMap.ToImmutable()));
}
finally
{
mIsRunning = false;
}
}
public async Task Complete()
{
lock (mLockObject)
{
if (mComplete)
{
throw new InvalidOperationException();
}
mComplete = true;
Monitor.PulseAll(mLockObject);
while (!mCompleteAck)
{
Monitor.Wait(mLockObject);
}
}
#if NET_4_6_2
// This will probably deadlock up to .NET 4.6.1 and might be fixed in .NET 4.6.2
// The cause is that Task.WhenAll does not respect TaskCreationOptions.RunContinuationsAsynchronously
await Task.WhenAll(mStorage.Select(x => x.GetCompletionTask())).ConfigureAwait(false);
#else
foreach (var stream in mStorage)
{
await stream.GetCompletionTask().ConfigureAwait(false);
}
#endif
int totalInputCount = 0;
var firstInputOffset = new int[mEncoders.Length];
for (int i = 0; i < mEncoders.Length; i++)
{
firstInputOffset[i] = totalInputCount;
totalInputCount += mDefinition.GetEncoder(i).InputCount;
}
var decoders = ImmutableArray.CreateBuilder <DecoderMetadata>(mEncoders.Length);
for (int i = 0; i < mEncoders.Length; i++)
{
var encoder = mDefinition.GetEncoder(i);
var settings = encoder.Settings;
var decoderType = settings.GetDecoderType();
var decoderInputCount = decoderType.GetInputCount();
var decoderInputs = ImmutableArray.CreateBuilder <DecoderInputMetadata>(decoderInputCount);
for (int j = 0; j < decoderInputCount; j++)
{
var encoderOutput = encoder.GetOutput(j).Target;
if (encoderOutput.IsStorage)
{
decoderInputs.Add(new DecoderInputMetadata(null, encoderOutput.Index));
}
else
{
decoderInputs.Add(new DecoderInputMetadata(encoderOutput.Node.Index, encoderOutput.Index));
}
}
var decoderOutputCount = decoderType.GetOutputCount();
var decoderOutputs = ImmutableArray.CreateBuilder <DecoderOutputMetadata>(decoderOutputCount);
for (int j = 0; j < decoderOutputCount; j++)
{
var encoderInput = encoder.GetInput(j).Source;
if (encoderInput.IsContent)
{
System.Diagnostics.Debug.Assert(mConnections[firstInputOffset[i] + j] == null);
decoderOutputs.Add(new DecoderOutputMetadata(mInput.GetFinalLength()));
}
else
{
decoderOutputs.Add(new DecoderOutputMetadata(mConnections[firstInputOffset[i] + j].GetFinalLength()));
}
}
decoders.Add(new DecoderMetadata(decoderType, settings.SerializeSettings(), decoderInputs.MoveToImmutable(), decoderOutputs.MoveToImmutable()));
}
var inputSource = mDefinition.GetContentSource().Target;
var definition = new ArchiveDecoderSection(
decoders.MoveToImmutable(),
new DecoderInputMetadata(inputSource.Node.Index, inputSource.Index),
mInput.GetFinalLength(),
mInput.GetFinalChecksum(),
mContent.ToImmutable());
mWriter.CompleteEncoderSession(this, mSection, definition, mStorage);
}
private static void SelectStream(
ArchiveMetadata archiveMetadata,
ArchiveDecoderSection sectionMetadata,
DecoderInputMetadata selector,
ReaderNode[] streams,
DecoderNode[] decoders,
out ReaderNode result,
out long length)
{
var decoderIndex = selector.DecoderIndex;
if (decoderIndex.HasValue)
{
length = sectionMetadata.Decoders[decoderIndex.Value].OutputStreams[selector.StreamIndex].Length;
result = decoders[decoderIndex.Value].GetOutputStream(selector.StreamIndex);
}
else
{
length = archiveMetadata.FileSections[selector.StreamIndex].Length;
result = streams[selector.StreamIndex];
}
}
/// <summary>
/// Creates a new iteration to unpack a sequence of streams from a decoded archive section.
/// </summary>
/// <param name="stream">A readable and seekable stream for the archive.</param>
/// <param name="metadata">The metadata for the archive.</param>
/// <param name="index">The decoder section index from the metadata which should be unpacked.</param>
/// <param name="password">An optional password to unpack the archive content.</param>
public DecodedSectionReader(Stream stream, ArchiveMetadata metadata, int index, PasswordStorage password)
{
mDecodedStream = new ArchiveSectionDecoder(stream, metadata, index, password);
mMetadata = metadata;
mDecoderSection = metadata.DecoderSections[index];
}
public async Task Complete()
{
lock (mLockObject)
{
if (mComplete)
throw new InvalidOperationException();
mComplete = true;
Monitor.PulseAll(mLockObject);
while (!mCompleteAck)
Monitor.Wait(mLockObject);
}
#if NET_4_6_2
// This will probably deadlock up to .NET 4.6.1 and might be fixed in .NET 4.6.2
// The cause is that Task.WhenAll does not respect TaskCreationOptions.RunContinuationsAsynchronously
await Task.WhenAll(mStorage.Select(x => x.GetCompletionTask())).ConfigureAwait(false);
#else
foreach (var stream in mStorage)
await stream.GetCompletionTask().ConfigureAwait(false);
#endif
int totalInputCount = 0;
var firstInputOffset = new int[mEncoders.Length];
for (int i = 0; i < mEncoders.Length; i++)
{
firstInputOffset[i] = totalInputCount;
totalInputCount += mDefinition.GetEncoder(i).InputCount;
}
var decoders = ImmutableArray.CreateBuilder<DecoderMetadata>(mEncoders.Length);
for (int i = 0; i < mEncoders.Length; i++)
{
var encoder = mDefinition.GetEncoder(i);
var settings = encoder.Settings;
var decoderType = settings.GetDecoderType();
var decoderInputCount = decoderType.GetInputCount();
var decoderInputs = ImmutableArray.CreateBuilder<DecoderInputMetadata>(decoderInputCount);
for (int j = 0; j < decoderInputCount; j++)
{
var encoderOutput = encoder.GetOutput(j).Target;
if (encoderOutput.IsStorage)
decoderInputs.Add(new DecoderInputMetadata(null, encoderOutput.Index));
else
decoderInputs.Add(new DecoderInputMetadata(encoderOutput.Node.Index, encoderOutput.Index));
}
var decoderOutputCount = decoderType.GetOutputCount();
var decoderOutputs = ImmutableArray.CreateBuilder<DecoderOutputMetadata>(decoderOutputCount);
for (int j = 0; j < decoderOutputCount; j++)
{
var encoderInput = encoder.GetInput(j).Source;
if (encoderInput.IsContent)
{
System.Diagnostics.Debug.Assert(mConnections[firstInputOffset[i] + j] == null);
decoderOutputs.Add(new DecoderOutputMetadata(mInput.GetFinalLength()));
}
else
{
decoderOutputs.Add(new DecoderOutputMetadata(mConnections[firstInputOffset[i] + j].GetFinalLength()));
}
}
decoders.Add(new DecoderMetadata(decoderType, settings.SerializeSettings(), decoderInputs.MoveToImmutable(), decoderOutputs.MoveToImmutable()));
}
var inputSource = mDefinition.GetContentSource().Target;
var definition = new ArchiveDecoderSection(
decoders.MoveToImmutable(),
new DecoderInputMetadata(inputSource.Node.Index, inputSource.Index),
mInput.GetFinalLength(),
mInput.GetFinalChecksum(),
mContent.ToImmutable());
mWriter.CompleteEncoderSession(this, mSection, definition, mStorage);
}
/// <summary>
/// Creates a new iteration to unpack a sequence of streams from a decoded archive section.
/// </summary>
/// <param name="stream">A readable and seekable stream for the archive.</param>
/// <param name="metadata">The metadata for the archive.</param>
/// <param name="index">The decoder section index from the metadata which should be unpacked.</param>
/// <param name="password">An optional password to unpack the archive content.</param>
public DecodedSectionReader(Stream stream, ArchiveMetadata metadata, int index, PasswordStorage password)
{
mDecodedStream = new ArchiveSectionDecoder(stream, metadata, index, password);
mMetadata = metadata;
mDecoderSection = metadata.DecoderSections[index];
}
private void WriteDecoderSection(ArchiveDecoderSection definition, ref int firstStreamIndex)
{
WriteNumber(definition.Decoders.Length);
var inputOffset = new int[definition.Decoders.Length];
var outputOffset = new int[definition.Decoders.Length];
for (int i = 1; i < definition.Decoders.Length; i++)
{
inputOffset[i] = inputOffset[i - 1] + definition.Decoders[i - 1].InputStreams.Length;
outputOffset[i] = outputOffset[i - 1] + definition.Decoders[i - 1].OutputStreams.Length;
}
for (int i = 0; i < definition.Decoders.Length; i++)
{
var decoder = definition.Decoders[i];
var id = decoder.DecoderType.Encode();
var multiStream = decoder.InputStreams.Length != 1 || decoder.OutputStreams.Length != 1;
var settings = decoder.Settings;
var hasSettings = !settings.IsDefaultOrEmpty;
System.Diagnostics.Debug.Assert(!id.IsDefaultOrEmpty && id.Length <= 15);
var flags = (byte)id.Length;
if (multiStream)
{
flags |= 0x10;
}
if (hasSettings)
{
flags |= 0x20;
}
WriteByte(flags);
foreach (var bt in id)
{
WriteByte(bt);
}
if (multiStream)
{
WriteNumber(decoder.InputStreams.Length);
WriteNumber(decoder.OutputStreams.Length);
}
if (hasSettings)
{
WriteNumber(settings.Length);
foreach (var bt in settings)
{
WriteByte(bt);
}
}
}
for (int i = 0; i < definition.Decoders.Length; i++)
{
var decoder = definition.Decoders[i];
for (int j = 0; j < decoder.InputStreams.Length; j++)
{
var input = decoder.InputStreams[j];
if (input.DecoderIndex.HasValue)
{
WriteNumber(inputOffset[i] + j);
WriteNumber(outputOffset[input.DecoderIndex.Value] + input.StreamIndex);
}
}
}
int fileStreamSections = 0;
foreach (var decoder in definition.Decoders)
{
foreach (var input in decoder.InputStreams)
{
if (!input.DecoderIndex.HasValue)
{
fileStreamSections += 1;
}
}
}
if (fileStreamSections > 1)
{
foreach (var decoder in definition.Decoders)
{
foreach (var input in decoder.InputStreams)
{
if (!input.DecoderIndex.HasValue)
{
WriteNumber(input.StreamIndex - firstStreamIndex);
}
}
}
}
firstStreamIndex += fileStreamSections;
}
