private static void SelectStream(
ArchiveMetadata archiveMetadata,
ArchiveDecoderSection sectionMetadata,
DecoderInputMetadata selector,
ReaderNode[] streams,
DecoderNode[] decoders,
out ReaderNode result,
out long length)
{
var decoderIndex = selector.DecoderIndex;
if (decoderIndex.HasValue)
{
length = sectionMetadata.Decoders[decoderIndex.Value].OutputStreams[selector.StreamIndex].Length;
result = decoders[decoderIndex.Value].GetOutputStream(selector.StreamIndex);
}
else
{
length = archiveMetadata.FileSections[selector.StreamIndex].Length;
result = streams[selector.StreamIndex];
}
}
private ArchiveSectionMetadataBuilder ReadSection(int rawInputStreamIndex)
{
var section = new ArchiveSectionMetadataBuilder();
int totalInputCount = 0;
int totalOutputCount = 0;
var decoderCount = ReadNumberAsInt32();
if (decoderCount == 0)
{
throw new InvalidDataException();
}
section.Decoders = new DecoderMetadataBuilder[decoderCount];
for (int i = 0; i < decoderCount; i++)
{
var decoder = ReadDecoder();
totalInputCount += decoder.InputCount;
totalOutputCount += decoder.OutputCount;
section.Decoders[i] = decoder;
}
// One output is the final output, the others need to be wired up.
var usedOutputMask = new bool[totalOutputCount];
for (int i = 1; i < totalOutputCount; i++)
{
int inputIndex = ReadNumberAsInt32();
int inputDecoderIndex = 0;
for (;;)
{
if (inputDecoderIndex == decoderCount)
{
throw new InvalidDataException();
}
if (inputIndex < section.Decoders[inputDecoderIndex].InputCount)
{
break;
}
inputIndex -= section.Decoders[inputDecoderIndex].OutputCount;
inputDecoderIndex += 1;
}
int outputIndex = ReadNumberAsInt32();
// Detect duplicate output connections through the output mask.
if (outputIndex >= totalOutputCount || usedOutputMask[outputIndex])
{
throw new InvalidDataException();
}
usedOutputMask[outputIndex] = true;
// Separate the output index into decoder index and stream index
int outputDecoderIndex = 0;
for (;;)
{
if (outputDecoderIndex == decoderCount)
{
throw new InvalidDataException();
}
var outputCount = section.Decoders[outputDecoderIndex].OutputCount;
if (outputIndex < outputCount)
{
break;
}
outputIndex -= outputCount;
outputDecoderIndex += 1;
}
// Detect duplicate input connections by checking for the placeholder.
if (section.Decoders[inputDecoderIndex].InputInfo[inputIndex].StreamIndex != Int32.MaxValue)
{
throw new InvalidDataException();
}
section.Decoders[inputDecoderIndex].InputInfo[inputIndex] = new DecoderInputMetadata(outputDecoderIndex, outputIndex);
}
bool foundFinalOutput = false;
for (int finalOutputIndex = 0, i = 0; i < decoderCount; i++)
{
var decoder = section.Decoders[i];
for (int j = 0; j < decoder.OutputCount; j++)
{
if (!usedOutputMask[finalOutputIndex++])
{
if (foundFinalOutput)
{
throw new InvalidDataException();
}
foundFinalOutput = true;
section.OutputStream = new DecoderInputMetadata(i, j);
}
}
}
if (!foundFinalOutput)
{
throw new InvalidDataException();
}
// Outputs must be wired up to unique inputs. Inputs which are not wired to outputs must be wired to raw streams.
// Note that negative overflow is not possible and positive overflow is ok in this calculation,
// it will fail the range check and trigger the exception, which is the intended behavior.
var requiredRawStreams = 1 + totalInputCount - totalOutputCount;
if (requiredRawStreams <= 0)
{
throw new InvalidDataException();
}
section.RequiredRawInputStreamCount = requiredRawStreams;
if (requiredRawStreams == 1)
{
bool connected = false;
foreach (var decoder in section.Decoders)
{
for (int i = 0; i < decoder.InputCount; i++)
{
if (decoder.InputInfo[i].StreamIndex == Int32.MaxValue)
{
if (connected)
{
throw new InvalidDataException();
}
connected = true;
decoder.InputInfo[i] = new DecoderInputMetadata(null, 0);
}
}
}
if (!connected)
{
throw new InvalidDataException();
}
}
else
{
for (int i = 0; i < requiredRawStreams; i++)
{
int inputIndex = ReadNumberAsInt32();
int inputDecoderIndex = 0;
for (;;)
{
if (inputDecoderIndex == decoderCount)
{
throw new InvalidDataException();
}
var decoder = section.Decoders[inputDecoderIndex];
if (inputIndex < decoder.InputCount)
{
break;
}
inputIndex -= decoder.OutputCount;
inputDecoderIndex += 1;
}
var decoderInput = section.Decoders[inputDecoderIndex].InputInfo;
if (decoderInput[inputIndex].StreamIndex != Int32.MaxValue)
{
throw new InvalidDataException();
}
decoderInput[inputIndex] = new DecoderInputMetadata(null, rawInputStreamIndex + i);
}
}
return(section);
}
private ArchiveSectionMetadataBuilder[] ReadSectionHeader(ImmutableArray <Stream> streams)
{
SkipToToken(ArchiveMetadataToken.Folder);
var sectionCount = ReadNumberAsInt32();
var sections = new ArchiveSectionMetadataBuilder[sectionCount];
int rawStreamCount = 0;
using (SelectStream(streams))
{
for (int i = 0; i < sectionCount; i++)
{
var section = ReadSection(rawStreamCount);
rawStreamCount += section.RequiredRawInputStreamCount;
sections[i] = section;
}
}
SkipToToken(ArchiveMetadataToken.CodersUnpackSize);
foreach (var section in sections)
{
for (int decoderIndex = 0; decoderIndex < section.Decoders.Length; decoderIndex++)
{
var decoder = section.Decoders[decoderIndex];
var outputCount = decoder.OutputCount;
var outputBuilder = ImmutableArray.CreateBuilder <DecoderOutputMetadata>(outputCount);
for (int outputIndex = 0; outputIndex < outputCount; outputIndex++)
{
var length = ReadNumberAsInt64();
outputBuilder.Add(new DecoderOutputMetadata(length));
var stream = new DecoderInputMetadata(decoderIndex, outputIndex);
if (section.OutputStream == stream)
{
section.OutputLength = length;
}
}
decoder.OutputInfo = outputBuilder;
}
}
for (;;)
{
var token = ReadToken();
if (token == ArchiveMetadataToken.End)
{
break;
}
if (token == ArchiveMetadataToken.CRC)
{
var vector = ReadOptionalBitVector(sectionCount);
for (int i = 0; i < sectionCount; i++)
{
if (vector[i])
{
sections[i].OutputChecksum = new Checksum(ReadInt32());
}
else
{
sections[i].OutputChecksum = null;
}
}
}
else
{
SkipDataBlock();
}
}
return(sections);
}
private DecoderMetadataBuilder ReadDecoder()
{
var decoder = new DecoderMetadataBuilder();
var mainByte = ReadByte();
if ((mainByte & 0x80) != 0)
{
throw new NotImplementedException();
}
var idLen = (mainByte & 0x0F);
if (idLen > 4)
{
throw new InvalidDataException(); // all known decoders use 4 bytes or less
}
int id = 0;
for (int i = idLen - 1; i >= 0; i--)
{
id |= ReadByte() << (i * 8);
}
decoder.Method = CompressionMethod.TryDecode(id);
if (decoder.Method.IsUndefined)
{
throw new InvalidDataException(); // unknown decoder
}
if ((mainByte & 0x10) != 0)
{
decoder.InputCount = ReadNumberAsInt32();
decoder.OutputCount = ReadNumberAsInt32();
}
else
{
decoder.InputCount = 1;
decoder.OutputCount = 1;
}
decoder.Method.CheckInputOutputCount(decoder.InputCount, decoder.OutputCount);
decoder.InputInfo = ImmutableArray.CreateBuilder <DecoderInputMetadata>(decoder.InputCount);
// We need to fill the array because it is not guaranteed that inputs will be connected in order.
// No valid decoder can have Int32.MaxValue inputs so we can use this as a placeholder value to detect bugs.
var placeholder = new DecoderInputMetadata(Int32.MaxValue, Int32.MaxValue);
for (int i = 0; i < decoder.InputCount; i++)
{
decoder.InputInfo.Add(placeholder);
}
if ((mainByte & 0x20) != 0)
{
var settingsLength = ReadNumberAsInt32();
var settingsBuilder = ImmutableArray.CreateBuilder <byte>(settingsLength);
for (int i = 0; i < settingsLength; i++)
{
settingsBuilder.Add(ReadByte());
}
decoder.Settings = settingsBuilder.MoveToImmutable();
}
else
{
decoder.Settings = ImmutableArray <byte> .Empty;
}
return(decoder);
}
private static void SelectStream(
ArchiveMetadata archiveMetadata,
ArchiveDecoderSection sectionMetadata,
DecoderInputMetadata selector,
ReaderNode[] streams,
DecoderNode[] decoders,
out ReaderNode result,
out long length)
{
var decoderIndex = selector.DecoderIndex;
if (decoderIndex.HasValue)
{
length = sectionMetadata.Decoders[decoderIndex.Value].OutputStreams[selector.StreamIndex].Length;
result = decoders[decoderIndex.Value].GetOutputStream(selector.StreamIndex);
}
else
{
length = archiveMetadata.FileSections[selector.StreamIndex].Length;
result = streams[selector.StreamIndex];
}
}
private DecoderMetadataBuilder ReadDecoder()
{
var decoder = new DecoderMetadataBuilder();
var mainByte = ReadByte();
#warning ??? why did we check this in the old decoder ???
if ((mainByte & 0x80) != 0)
throw new NotImplementedException();
var idLen = (mainByte & 0x0F);
if (idLen > 4)
throw new InvalidDataException(); // all known decoders use 4 bytes or less
int id = 0;
for (int i = idLen - 1; i >= 0; i--)
id |= ReadByte() << (i * 8);
decoder.Method = CompressionMethod.TryDecode(id);
if (decoder.Method.IsUndefined)
throw new InvalidDataException(); // unknown decoder
if ((mainByte & 0x10) != 0)
{
decoder.InputCount = ReadNumberAsInt32();
decoder.OutputCount = ReadNumberAsInt32();
}
else
{
decoder.InputCount = 1;
decoder.OutputCount = 1;
}
decoder.Method.CheckInputOutputCount(decoder.InputCount, decoder.OutputCount);
decoder.InputInfo = ImmutableArray.CreateBuilder<DecoderInputMetadata>(decoder.InputCount);
// We need to fill the array because it is not guaranteed that inputs will be connected in order.
// No valid decoder can have Int32.MaxValue inputs so we can use this as a placeholder value to detect bugs.
var placeholder = new DecoderInputMetadata(Int32.MaxValue, Int32.MaxValue);
for (int i = 0; i < decoder.InputCount; i++)
decoder.InputInfo.Add(placeholder);
if ((mainByte & 0x20) != 0)
{
var settingsLength = ReadNumberAsInt32();
var settingsBuilder = ImmutableArray.CreateBuilder<byte>(settingsLength);
for (int i = 0; i < settingsLength; i++)
settingsBuilder.Add(ReadByte());
decoder.Settings = settingsBuilder.MoveToImmutable();
}
else
{
decoder.Settings = ImmutableArray<byte>.Empty;
}
return decoder;
}
private ArchiveSectionMetadataBuilder ReadSection(int rawInputStreamIndex)
{
var section = new ArchiveSectionMetadataBuilder();
int totalInputCount = 0;
int totalOutputCount = 0;
var decoderCount = ReadNumberAsInt32();
if (decoderCount == 0)
throw new InvalidDataException();
section.Decoders = new DecoderMetadataBuilder[decoderCount];
for (int i = 0; i < decoderCount; i++)
{
var decoder = ReadDecoder();
totalInputCount += decoder.InputCount;
totalOutputCount += decoder.OutputCount;
section.Decoders[i] = decoder;
}
// One output is the final output, the others need to be wired up.
var usedOutputMask = new bool[totalOutputCount];
for (int i = 1; i < totalOutputCount; i++)
{
int inputIndex = ReadNumberAsInt32();
int inputDecoderIndex = 0;
for (;;)
{
if (inputDecoderIndex == decoderCount)
throw new InvalidDataException();
if (inputIndex < section.Decoders[inputDecoderIndex].InputCount)
break;
inputIndex -= section.Decoders[inputDecoderIndex].OutputCount;
inputDecoderIndex += 1;
}
int outputIndex = ReadNumberAsInt32();
// Detect duplicate output connections through the output mask.
if (outputIndex >= totalOutputCount || usedOutputMask[outputIndex])
throw new InvalidDataException();
usedOutputMask[outputIndex] = true;
// Separate the output index into decoder index and stream index
int outputDecoderIndex = 0;
for (;;)
{
if (outputDecoderIndex == decoderCount)
throw new InvalidDataException();
var outputCount = section.Decoders[outputDecoderIndex].OutputCount;
if (outputIndex < outputCount)
break;
outputIndex -= outputCount;
outputDecoderIndex += 1;
}
// Detect duplicate input connections by checking for the placeholder.
if (section.Decoders[inputDecoderIndex].InputInfo[inputIndex].StreamIndex != Int32.MaxValue)
throw new InvalidDataException();
section.Decoders[inputDecoderIndex].InputInfo[inputIndex] = new DecoderInputMetadata(outputDecoderIndex, outputIndex);
}
bool foundFinalOutput = false;
for (int finalOutputIndex = 0, i = 0; i < decoderCount; i++)
{
var decoder = section.Decoders[i];
for (int j = 0; j < decoder.OutputCount; j++)
{
if (!usedOutputMask[finalOutputIndex++])
{
if (foundFinalOutput)
throw new InvalidDataException();
foundFinalOutput = true;
section.OutputStream = new DecoderInputMetadata(i, j);
}
}
}
if (!foundFinalOutput)
throw new InvalidDataException();
// Outputs must be wired up to unique inputs. Inputs which are not wired to outputs must be wired to raw streams.
// Note that negative overflow is not possible and positive overflow is ok in this calculation,
// it will fail the range check and trigger the exception, which is the intended behavior.
var requiredRawStreams = 1 + totalInputCount - totalOutputCount;
if (requiredRawStreams <= 0)
throw new InvalidDataException();
section.RequiredRawInputStreamCount = requiredRawStreams;
if (requiredRawStreams == 1)
{
bool connected = false;
foreach (var decoder in section.Decoders)
{
for (int i = 0; i < decoder.InputCount; i++)
{
if (decoder.InputInfo[i].StreamIndex == Int32.MaxValue)
{
if (connected)
throw new InvalidDataException();
connected = true;
decoder.InputInfo[i] = new DecoderInputMetadata(null, 0);
}
}
}
if (!connected)
throw new InvalidDataException();
}
else
{
for (int i = 0; i < requiredRawStreams; i++)
{
int inputIndex = ReadNumberAsInt32();
int inputDecoderIndex = 0;
for (;;)
{
if (inputDecoderIndex == decoderCount)
throw new InvalidDataException();
var decoder = section.Decoders[inputDecoderIndex];
if (inputIndex < decoder.InputCount)
break;
inputIndex -= decoder.OutputCount;
inputDecoderIndex += 1;
}
var decoderInput = section.Decoders[inputDecoderIndex].InputInfo;
if (decoderInput[inputIndex].StreamIndex != Int32.MaxValue)
throw new InvalidDataException();
decoderInput[inputIndex] = new DecoderInputMetadata(null, rawInputStreamIndex + i);
}
}
return section;
}
private ArchiveSectionMetadataBuilder[] ReadSectionHeader(ImmutableArray<Stream> streams)
{
SkipToToken(ArchiveMetadataToken.Folder);
var sectionCount = ReadNumberAsInt32();
var sections = new ArchiveSectionMetadataBuilder[sectionCount];
int rawStreamCount = 0;
using (SelectStream(streams))
{
for (int i = 0; i < sectionCount; i++)
{
var section = ReadSection(rawStreamCount);
rawStreamCount += section.RequiredRawInputStreamCount;
sections[i] = section;
}
}
SkipToToken(ArchiveMetadataToken.CodersUnpackSize);
foreach (var section in sections)
{
for (int decoderIndex = 0; decoderIndex < section.Decoders.Length; decoderIndex++)
{
var decoder = section.Decoders[decoderIndex];
var outputCount = decoder.OutputCount;
var outputBuilder = ImmutableArray.CreateBuilder<DecoderOutputMetadata>(outputCount);
for (int outputIndex = 0; outputIndex < outputCount; outputIndex++)
{
var length = ReadNumberAsInt64();
outputBuilder.Add(new DecoderOutputMetadata(length));
var stream = new DecoderInputMetadata(decoderIndex, outputIndex);
if (section.OutputStream == stream)
section.OutputLength = length;
}
decoder.OutputInfo = outputBuilder;
}
}
for (;;)
{
var token = ReadToken();
if (token == ArchiveMetadataToken.End)
break;
if (token == ArchiveMetadataToken.CRC)
{
var vector = ReadOptionalBitVector(sectionCount);
for (int i = 0; i < sectionCount; i++)
{
if (vector[i])
sections[i].OutputChecksum = new Checksum(ReadInt32());
else
sections[i].OutputChecksum = null;
}
}
else
{
SkipDataBlock();
}
}
return sections;
}
