internal static Stream GetPayloadStream(Stream stream)
{
using (ArFile archive = new ArFile(stream, leaveOpen: true))
{
while (archive.Read())
{
if (archive.FileName.StartsWith("data.tar."))
{
var ext = Path.GetExtension(archive.FileName);
if (ext == ".gz")
{
return(new GZipDecompressor(archive.Open(), false));
}
if (ext == ".xz")
{
// For some reason it complains about corrupted data when we try to read using smaller chunks
var payload = new MemoryStream();
using (var xz = new XZInputStream(archive.Open()))
{
xz.CopyTo(payload);
payload.Seek(0, SeekOrigin.Begin);
return(payload);
}
}
throw new InvalidDataException("Don't know how to decompress " + archive.FileName);
}
}
throw new InvalidDataException("data.tar.?? not found");
}
}
public EncodedFileInfo(Script p, string dirName, string fileName)
{
string section = $"EncodedFile-{dirName}-{fileName}";
if (p.Sections.ContainsKey(section) == false)
{
throw new FileDecodeFailException($"[{dirName}\\{fileName}] does not exists in [{p.FullPath}]");
}
List <string> encodedList = p.Sections[$"EncodedFile-{dirName}-{fileName}"].GetLinesOnce();
if (Ini.GetKeyValueFromLine(encodedList[0], out string key, out string value))
{
throw new FileDecodeFailException("Encoded lines are malformed");
}
// [Stage 1] Concat sliced base64-encoded lines into one string
byte[] decoded;
{
int.TryParse(value, out int blockCount);
encodedList.RemoveAt(0); // Remove "lines=n"
// Each line is 64KB block
if (Ini.GetKeyValueFromLines(encodedList, out List <string> keys, out List <string> base64Blocks))
{
throw new FileDecodeFailException("Encoded lines are malformed");
}
StringBuilder b = new StringBuilder();
foreach (string block in base64Blocks)
{
b.Append(block);
}
switch (b.Length % 4)
{
case 0:
break;
case 1:
throw new FileDecodeFailException("Encoded lines are malformed");
case 2:
b.Append("==");
break;
case 3:
b.Append("=");
break;
}
decoded = Convert.FromBase64String(b.ToString());
}
// [Stage 2] Read final footer
const int finalFooterLen = 0x24;
int finalFooterIdx = decoded.Length - finalFooterLen;
// 0x00 - 0x04 : 4B -> CRC32
uint full_crc32 = BitConverter.ToUInt32(decoded, finalFooterIdx + 0x00);
// 0x0C - 0x0F : 4B -> Zlib Compressed Footer Length
int compressedFooterLen = (int)BitConverter.ToUInt32(decoded, finalFooterIdx + 0x0C);
int compressedFooterIdx = decoded.Length - (finalFooterLen + compressedFooterLen);
// 0x10 - 0x17 : 8B -> Zlib Compressed File Length
int compressedBodyLen = (int)BitConverter.ToUInt64(decoded, finalFooterIdx + 0x10);
// [Stage 3] Validate final footer
this.FinalFooterValid = true;
if (compressedBodyLen != compressedFooterIdx)
{
this.FinalFooterValid = false;
}
uint calcFull_crc32 = Crc32Checksum.Crc32(decoded, 0, finalFooterIdx);
if (full_crc32 != calcFull_crc32)
{
this.FinalFooterValid = false;
}
if (this.FinalFooterValid == false)
{
return;
}
// [Stage 4] Decompress first footer
byte[] rawFooter;
using (MemoryStream rawFooterStream = new MemoryStream())
{
using (MemoryStream ms = new MemoryStream(decoded, compressedFooterIdx, compressedFooterLen))
using (ZLibStream zs = new ZLibStream(ms, CompressionMode.Decompress, CompressionLevel.Default))
{
zs.CopyTo(rawFooterStream);
}
rawFooter = rawFooterStream.ToArray();
}
// [Stage 5] Read first footer
this.FirstFooterValid = true;
// 0x200 - 0x207 : 8B -> Length of raw file, in little endian
int rawBodyLen = (int)BitConverter.ToUInt32(rawFooter, 0x200);
// 0x208 - 0x20F : 8B -> Length of zlib-compressed file, in little endian
// Note: In Type 2, 0x208 entry is null - padded
int compressedBodyLen2 = (int)BitConverter.ToUInt32(rawFooter, 0x208);
// 0x220 - 0x223 : 4B -> CRC32C Checksum of zlib-compressed file
uint compressedBody_crc32 = BitConverter.ToUInt32(rawFooter, 0x220);
// 0x224 : 1B -> Compress Mode (Type 1 : 00, Type 2 : 01)
byte compMode = rawFooter[0x224];
// 0x225 : 1B -> ZLib Compress Level (Type 1 : 01~09, Type 2 : 00)
byte compLevel = rawFooter[0x225];
// [Stage 6] Validate first footer
if (compMode == 0)
{
this.Mode = EncodedFile.EncodeMode.ZLib;
if (compLevel < 1 || 9 < compLevel)
{
this.FirstFooterValid = false;
}
if (compressedBodyLen2 == 0 || (compressedBodyLen2 != compressedBodyLen))
{
this.FirstFooterValid = false;
}
}
else if (compMode == 1)
{
this.Mode = EncodedFile.EncodeMode.Raw;
if (compLevel != 0)
{
this.FirstFooterValid = false;
}
if (compressedBodyLen2 != 0)
{
this.FirstFooterValid = false;
}
}
else // Wrong compMode
{
this.FirstFooterValid = false;
}
if (this.FirstFooterValid == false)
{
return;
}
// [Stage 7] Decompress body
switch ((EncodedFile.EncodeMode)compMode)
{
case EncodedFile.EncodeMode.ZLib:
{
this.RawBodyStream = new MemoryStream();
using (MemoryStream ms = new MemoryStream(decoded, 0, compressedBodyLen))
using (ZLibStream zs = new ZLibStream(ms, CompressionMode.Decompress, CompressionLevel.Default))
{
zs.CopyTo(this.RawBodyStream);
}
this.CompressedBodyValid = true;
}
break;
case EncodedFile.EncodeMode.Raw:
{
this.RawBodyStream = new MemoryStream(decoded, 0, rawBodyLen);
this.CompressedBodyValid = true;
}
break;
case EncodedFile.EncodeMode.XZ:
{
using (MemoryStream ms = new MemoryStream(decoded, 0, compressedBodyLen))
using (XZInputStream xzs = new XZInputStream(ms))
{
xzs.CopyTo(this.RawBodyStream);
}
this.CompressedBodyValid = true;
}
break;
default:
throw new InternalException($"Wrong EncodeMode [{compMode}]");
}
this.RawBodyStream.Position = 0;
// [Stage 8] Validate decompressed body
this.RawBodyValid = true;
uint calcCompBody_crc32 = Crc32Checksum.Crc32(RawBodyStream.ToArray());
if (compressedBody_crc32 != calcCompBody_crc32)
{
this.RawBodyValid = false;
}
// [Stage 9] Return decompressed body stream
this.RawBodyStream.Position = 0;
}
private static MemoryStream Decode(List <string> encodedList)
{
if (Ini.GetKeyValueFromLine(encodedList[0], out string key, out string value))
{
throw new FileDecodeFailException("Encoded lines are malformed");
}
// [Stage 1] Concat sliced base64-encoded lines into one string
byte[] decoded;
{
int.TryParse(value, out int blockCount);
encodedList.RemoveAt(0); // Remove "lines=n"
// Each line is 64KB block
if (Ini.GetKeyValueFromLines(encodedList, out List <string> keys, out List <string> base64Blocks))
{
throw new FileDecodeFailException("Encoded lines are malformed");
}
StringBuilder b = new StringBuilder();
foreach (string block in base64Blocks)
{
b.Append(block);
}
switch (b.Length % 4)
{
case 0:
break;
case 1:
throw new FileDecodeFailException("Encoded lines are malformed");
case 2:
b.Append("==");
break;
case 3:
b.Append("=");
break;
}
decoded = Convert.FromBase64String(b.ToString());
}
// [Stage 2] Read final footer
const int finalFooterLen = 0x24;
int finalFooterIdx = decoded.Length - finalFooterLen;
// 0x00 - 0x04 : 4B -> CRC32
uint full_crc32 = BitConverter.ToUInt32(decoded, finalFooterIdx + 0x00);
// 0x0C - 0x0F : 4B -> Zlib Compressed Footer Length
int compressedFooterLen = (int)BitConverter.ToUInt32(decoded, finalFooterIdx + 0x0C);
int compressedFooterIdx = decoded.Length - (finalFooterLen + compressedFooterLen);
// 0x10 - 0x17 : 8B -> Zlib Compressed File Length
int compressedBodyLen = (int)BitConverter.ToUInt64(decoded, finalFooterIdx + 0x10);
// [Stage 3] Validate final footer
if (compressedBodyLen != compressedFooterIdx)
{
throw new FileDecodeFailException($"Encoded file is corrupted");
}
uint calcFull_crc32 = Crc32Checksum.Crc32(decoded, 0, finalFooterIdx);
if (full_crc32 != calcFull_crc32)
{
throw new FileDecodeFailException($"Encoded file is corrupted");
}
// [Stage 4] Decompress first footer
byte[] rawFooter;
using (MemoryStream rawFooterStream = new MemoryStream())
{
using (MemoryStream ms = new MemoryStream(decoded, compressedFooterIdx, compressedFooterLen))
using (ZLibStream zs = new ZLibStream(ms, CompressionMode.Decompress, CompressionLevel.Default))
{
zs.CopyTo(rawFooterStream);
}
rawFooter = rawFooterStream.ToArray();
}
// [Stage 5] Read first footer
// 0x200 - 0x207 : 8B -> Length of raw file, in little endian
int rawBodyLen = BitConverter.ToInt32(rawFooter, 0x200);
// 0x208 - 0x20F : 8B -> Length of zlib-compressed file, in little endian
// Note: In Type 2, 0x208 entry is null - padded
int compressedBodyLen2 = BitConverter.ToInt32(rawFooter, 0x208);
// 0x220 - 0x223 : 4B -> CRC32C Checksum of zlib-compressed file
uint compressedBody_crc32 = BitConverter.ToUInt32(rawFooter, 0x220);
// 0x224 : 1B -> Compress Mode (Type 1 : 00, Type 2 : 01)
byte compMode = rawFooter[0x224];
// 0x225 : 1B -> ZLib Compress Level (Type 1 : 01~09, Type 2 : 00)
byte compLevel = rawFooter[0x225];
// [Stage 6] Validate first footer
switch ((EncodeMode)compMode)
{
case EncodeMode.ZLib: // Type 1, zlib
{
if (compressedBodyLen2 == 0 || (compressedBodyLen2 != compressedBodyLen))
{
throw new FileDecodeFailException($"Encoded file is corrupted: compMode");
}
if (compLevel < 1 || 9 < compLevel)
{
throw new FileDecodeFailException($"Encoded file is corrupted: compLevel");
}
}
break;
case EncodeMode.Raw: // Type 2, raw
{
if (compressedBodyLen2 != 0)
{
throw new FileDecodeFailException($"Encoded file is corrupted: compMode");
}
if (compLevel != 0)
{
throw new FileDecodeFailException($"Encoded file is corrupted: compLevel");
}
}
break;
case EncodeMode.XZ: // Type 3, LZMA
{
if (compressedBodyLen2 == 0 || (compressedBodyLen2 != compressedBodyLen))
{
throw new FileDecodeFailException($"Encoded file is corrupted: compMode");
}
if (compLevel < 1 || 9 < compLevel)
{
throw new FileDecodeFailException($"Encoded file is corrupted: compLevel");
}
}
break;
default:
throw new FileDecodeFailException($"Encoded file is corrupted: compMode");
}
// [Stage 7] Decompress body
MemoryStream rawBodyStream = new MemoryStream(); // This stream should be alive even after this method returns
switch ((EncodeMode)compMode)
{
case EncodeMode.ZLib: // Type 1, zlib
{
using (MemoryStream ms = new MemoryStream(decoded, 0, compressedBodyLen))
using (ZLibStream zs = new ZLibStream(ms, CompressionMode.Decompress, false))
{
zs.CopyTo(rawBodyStream);
}
}
break;
case EncodeMode.Raw: // Type 2, raw
{
// rawBodyStream = new MemoryStream(decoded, 0, rawBodyLen);
rawBodyStream.Write(decoded, 0, rawBodyLen);
}
break;
case EncodeMode.XZ: // Type 3, LZMA
{
using (MemoryStream ms = new MemoryStream(decoded, 0, compressedBodyLen))
using (XZInputStream xzs = new XZInputStream(ms))
{
xzs.CopyTo(rawBodyStream);
}
}
break;
default:
throw new FileDecodeFailException($"Encoded file is corrupted: compMode");
}
rawBodyStream.Position = 0;
// [Stage 8] Validate decompressed body
uint calcCompBody_crc32 = Crc32Checksum.Crc32(rawBodyStream.ToArray());
if (compressedBody_crc32 != calcCompBody_crc32)
{
throw new FileDecodeFailException($"Encoded file is corrupted: body");
}
// [Stage 9] Return decompressed body stream
rawBodyStream.Position = 0;
return(rawBodyStream);
}
private static void InitializeCardsManager()
{
manager = new CardsManager();
if (!File.Exists(cache_data_file))
{
max_steps += 2;
if (!File.Exists(cache_data_printings_file))
{
max_steps += 3;
if (!File.Exists(cache_data_printings_archive))
{
max_steps += 2;
}
if (!File.Exists(cache_data_tokens_file))
{
max_steps += 2;
}
}
}
if (!File.Exists(cache_data_file))
{
Console.WriteLine(GetSteps() + "Cache (" + cache_data_file + ") not found, creating cache...");
if (!File.Exists(cache_data_printings_file))
{
Console.WriteLine(GetSteps() + "Card data (" + cache_data_printings_file + ") not found...");
if (!File.Exists(cache_data_printings_archive))
{
Console.WriteLine(GetSteps() + "Card archive (" + cache_data_printings_archive + ") not found, downloading...");
Download(allsets_url, cache_data_printings_archive, true);
Console.WriteLine(GetSteps() + "Download complete");
}
if (!File.Exists(cache_data_tokens_file))
{
Console.WriteLine(GetSteps() + "Token XML (" + cache_data_tokens_file + ") not found, downloading...");
Download(tokens_url, cache_data_tokens_file);
Console.WriteLine(GetSteps() + "Download complete");
}
Console.WriteLine(GetSteps() + "Extracting card archive...");
using (Stream xz = new XZInputStream(File.OpenRead(cache_data_printings_archive)))
using (Stream stream = new FileStream(cache_data_printings_file, FileMode.OpenOrCreate))
{
xz.CopyTo(stream);
}
Console.WriteLine(GetSteps() + "Extraction complete");
}
Console.WriteLine(GetSteps() + "Processing data set (cards and tokens)...");
string string_data = File.ReadAllText(cache_data_printings_file);
JObject data = JObject.Parse(string_data);
List <Thread> threads = new List <Thread>();
foreach (var set in data)
{
threads.Add(new Thread(() =>
{
Set.Parse(set.Key, set.Value.ToObject <JObject>(), manager);
}));
}
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
WriteToBinaryFile(cache_data_file, manager);
Console.WriteLine(GetSteps() + "Cache (" + cache_data_file + ") creation completed.");
}
else
{
Console.Write(GetSteps() + "Cache (" + cache_data_file + ") found! Loading...");
manager = ReadFromBinaryFile <CardsManager>(cache_data_file);
Console.WriteLine(" Done");
}
}