/// <summary>
/// Reads the header intro table of contents from an input stream.
/// </summary>
/// <param name="stream">The stream to read from.</param>
/// <param name="offset">
/// The offset to the header intro table of contents from the beginning of the stream.
/// </param>
/// <param name="p">Progress info.</param>
/// <returns>The loaded header intro offets data.</returns>
internal async Task <Nefs20HeaderIntroToc> Read20HeaderIntroTocAsync(Stream stream, uint offset, NefsProgress p)
{
var toc = new Nefs20HeaderIntroToc();
await FileData.ReadDataAsync(stream, offset, toc, NefsVersion.Version200, p);
return(toc);
}
/// <summary>
/// Writes the header intro table of contents to an output stream.
/// </summary>
/// <param name="stream">The stream to write to.</param>
/// <param name="offset">The absolute offset in the stream to write at.</param>
/// <param name="toc">The table of contents to write.</param>
/// <param name="p">Progress info.</param>
/// <returns>An async task.</returns>
internal async Task WriteHeaderIntroTocAsync(Stream stream, UInt64 offset, Nefs20HeaderIntroToc toc, NefsProgress p)
{
using (var t = p.BeginTask(1.0f))
{
await FileData.WriteDataAsync(stream, offset, toc, NefsVersion.Version200, p);
}
}
/// <summary>
/// Creates a test archive. Does not write an archive to disk. Just creates a <see
/// cref="NefsArchive"/> object.
/// </summary>
/// <param name="filePath">The file path to use for the archive.</param>
/// <returns>A <see cref="NefsArchive"/>.</returns>
public static NefsArchive Create(string filePath)
{
var items = new NefsItemList(filePath);
Assert.Empty(items.EnumerateById());
Assert.Empty(items.EnumerateDepthFirstByName());
var intro = new NefsHeaderIntro();
intro.Data0x6c_NumberOfItems.Value = (uint)items.Count;
var toc = new Nefs20HeaderIntroToc();
var header = new Nefs20Header(intro, toc, items);
return(new NefsArchive(header, items));
}
/// <summary>
/// Creates a test archive. Does not write an archive to disk. Just creates a <see
/// cref="NefsArchive"/> object.
/// </summary>
/// <param name="filePath">The file path to use for the archive.</param>
/// <returns>A <see cref="NefsArchive"/>.</returns>
public static NefsArchive Create(string filePath)
{
var items = new NefsItemList(filePath);
var aesString = "44927647059D3D73CDCC8D4C6E808538CAD7622D076A507E16C43A8DD8E3B5AB";
var file1Attributes = new NefsItemAttributes(v20IsZlib: true);
var file1Chunks = NefsDataChunk.CreateChunkList(File1ChunkSizes, TestHelpers.TestTransform);
var file1DataSource = new NefsItemListDataSource(items, File1Offset, new NefsItemSize(File1ExtractedSize, file1Chunks));
var file1 = new NefsItem(File1Guid, new NefsItemId(File1ItemId), File1Name, new NefsItemId(File1DirectoryId), file1DataSource, TestHelpers.TestTransform, file1Attributes);
items.Add(file1);
var dir1Attributes = new NefsItemAttributes(isDirectory: true);
var dir1DataSource = new NefsEmptyDataSource();
var dir1 = new NefsItem(Dir1Guid, new NefsItemId(Dir1ItemId), Dir1Name, new NefsItemId(Dir1DirectoryId), dir1DataSource, null, dir1Attributes);
items.Add(dir1);
var file2Attributes = new NefsItemAttributes(v20IsZlib: true);
var file2Chunks = NefsDataChunk.CreateChunkList(File2ChunkSizes, TestHelpers.TestTransform);
var file2DataSource = new NefsItemListDataSource(items, File2Offset, new NefsItemSize(File2ExtractedSize, file2Chunks));
var file2 = new NefsItem(File2Guid, new NefsItemId(File2ItemId), File2Name, new NefsItemId(File2DirectoryId), file2DataSource, TestHelpers.TestTransform, file2Attributes);
items.Add(file2);
var file3Attributes = new NefsItemAttributes(v20IsZlib: true);
var file3Transform = new NefsDataTransform(File3ExtractedSize);
var file3Chunks = NefsDataChunk.CreateChunkList(File3ChunkSizes, file3Transform);
var file3DataSource = new NefsItemListDataSource(items, File3Offset, new NefsItemSize(File3ExtractedSize, file3Chunks));
var file3 = new NefsItem(File3Guid, new NefsItemId(File3ItemId), File3Name, new NefsItemId(File3DirectoryId), file3DataSource, file3Transform, file3Attributes);
items.Add(file3);
Assert.Equal((int)NumItems, items.Count);
var intro = new NefsHeaderIntro();
intro.Data0x6c_NumberOfItems.Value = (uint)items.Count;
intro.Data0x24_AesKeyHexString.Value = Encoding.ASCII.GetBytes(aesString);
var toc = new Nefs20HeaderIntroToc();
var header = new Nefs20Header(intro, toc, items);
return(new NefsArchive(header, items));
}
/// <summary>
/// Creates a test archive. Does not write an archive to disk. Just creates a <see
/// cref="NefsArchive"/> object.
/// </summary>
/// <param name="filePath">The file path to use for the archive.</param>
/// <returns>A <see cref="NefsArchive"/>.</returns>
public static NefsArchive Create(string filePath)
{
var items = new NefsItemList(filePath);
var file1Attributes = new NefsItemAttributes(v20IsZlib: true);
var file1Chunks = NefsDataChunk.CreateChunkList(File1ChunkSizes, TestHelpers.TestTransform);
var file1DataSource = new NefsItemListDataSource(items, File1Offset, new NefsItemSize(File1ExtractedSize, file1Chunks));
var file1 = new NefsItem(File1Guid, new NefsItemId(File1ItemId), File1Name, new NefsItemId(File1DirectoryId), file1DataSource, TestHelpers.TestTransform, file1Attributes);
items.Add(file1);
var dir1Attributes = new NefsItemAttributes(isDirectory: true);
var dir1DataSource = new NefsEmptyDataSource();
var dir1 = new NefsItem(Dir1Guid, new NefsItemId(Dir1ItemId), Dir1Name, new NefsItemId(Dir1DirectoryId), dir1DataSource, null, dir1Attributes);
items.Add(dir1);
var file2Attributes = new NefsItemAttributes(v20IsZlib: true);
var file2Chunks = NefsDataChunk.CreateChunkList(File2ChunkSizes, TestHelpers.TestTransform);
var file2DataSource = new NefsItemListDataSource(items, File2Offset, new NefsItemSize(File2ExtractedSize, file2Chunks));
var file2 = new NefsItem(File2Guid, new NefsItemId(File2ItemId), File2Name, new NefsItemId(File2DirectoryId), file2DataSource, TestHelpers.TestTransform, file2Attributes);
items.Add(file2);
var file3Attributes = new NefsItemAttributes(v20IsZlib: true);
var file3Chunks = NefsDataChunk.CreateChunkList(File3ChunkSizes, TestHelpers.TestTransform);
var file3DataSource = new NefsItemListDataSource(items, File3Offset, new NefsItemSize(File3ExtractedSize, file3Chunks));
var file3 = new NefsItem(File3Guid, new NefsItemId(File3ItemId), File3Name, new NefsItemId(File3DirectoryId), file3DataSource, TestHelpers.TestTransform, file3Attributes);
items.Add(file3);
Assert.Equal((int)NumItems, items.Count);
var intro = new NefsHeaderIntro();
intro.Data0x6c_NumberOfItems.Value = (uint)items.Count;
var toc = new Nefs20HeaderIntroToc();
var header = new Nefs20Header(intro, toc, items);
return(new NefsArchive(header, items));
}
/// <summary>
/// Creates a <see cref="NefsArchive"/> to be used for testing.
/// </summary>
/// <param name="filePath">The file path to associate with the archive.</param>
/// <returns>An archive object.</returns>
/// <remarks><![CDATA[ Test archive items: /file1 /dir1 /dir1/file2 ]]></remarks>
internal static NefsArchive CreateTestArchive(string filePath)
{
var items = new NefsItemList(filePath);
var transform = new NefsDataTransform(50, true);
var file1Attributes = new NefsItemAttributes(v20IsZlib: true);
var file1Chunks = NefsDataChunk.CreateChunkList(new List <UInt32> {
2, 3, 4
}, transform);
var file1DataSource = new NefsItemListDataSource(items, 100, new NefsItemSize(20, file1Chunks));
var file1 = new NefsItem(Guid.NewGuid(), new NefsItemId(0), "file1", new NefsItemId(0), file1DataSource, transform, file1Attributes);
items.Add(file1);
var dir1Attributes = new NefsItemAttributes(isDirectory: true);
var dir1DataSource = new NefsEmptyDataSource();
var dir1 = new NefsItem(Guid.NewGuid(), new NefsItemId(1), "dir1", new NefsItemId(1), dir1DataSource, null, dir1Attributes);
items.Add(dir1);
var file2Attributes = new NefsItemAttributes(v20IsZlib: true);
var file2Chunks = NefsDataChunk.CreateChunkList(new List <UInt32> {
5, 6, 7
}, transform);
var file2DataSource = new NefsItemListDataSource(items, 104, new NefsItemSize(15, file2Chunks));
var file2 = new NefsItem(Guid.NewGuid(), new NefsItemId(2), "file2", dir1.Id, file2DataSource, transform, file2Attributes);
items.Add(file2);
var intro = new NefsHeaderIntro();
var toc = new Nefs20HeaderIntroToc();
var header = new Nefs20Header(intro, toc, items);
return(new NefsArchive(header, items));
}
/// <summary>
/// Reads a version 2.0 header from an input stream.
/// </summary>
/// <param name="stream">The stream to read from.</param>
/// <param name="offset">The offset to the header from the beginning of the stream.</param>
/// <param name="part6Stream">The stream that contains part 6/7 data.</param>
/// <param name="part6Offset">The offset to the start of part 6/7 data.</param>
/// <param name="intro">The pre-parsed header intro.</param>
/// <param name="p">Progress info.</param>
/// <returns>The loaded header.</returns>
internal async Task <Nefs20Header> Read20HeaderAsync(
Stream stream,
ulong offset,
Stream part6Stream,
ulong part6Offset,
NefsHeaderIntro intro,
NefsProgress p)
{
Nefs20HeaderIntroToc toc = null;
NefsHeaderPart1 part1 = null;
NefsHeaderPart2 part2 = null;
NefsHeaderPart3 part3 = null;
Nefs20HeaderPart4 part4 = null;
NefsHeaderPart5 part5 = null;
Nefs20HeaderPart6 part6 = null;
NefsHeaderPart7 part7 = null;
NefsHeaderPart8 part8 = null;
// Calc weight of each task (8 parts + table of contents)
var weight = 1.0f / 10.0f;
using (p.BeginTask(weight, "Reading header intro table of contents"))
{
toc = await this.Read20HeaderIntroTocAsync(stream, Nefs20HeaderIntroToc.Offset, p);
}
using (p.BeginTask(weight, "Reading header part 1"))
{
part1 = await this.ReadHeaderPart1Async(stream, toc.OffsetToPart1, toc.Part1Size, p);
}
using (p.BeginTask(weight, "Reading header part 2"))
{
part2 = await this.ReadHeaderPart2Async(stream, toc.OffsetToPart2, toc.Part2Size, p);
}
using (p.BeginTask(weight, "Reading header part 3"))
{
part3 = await this.ReadHeaderPart3Async(stream, toc.OffsetToPart3, toc.Part3Size, p);
}
using (p.BeginTask(weight, "Reading header part 4"))
{
part4 = await this.Read20HeaderPart4Async(stream, toc.OffsetToPart4, toc.Part4Size, part1, p);
}
using (p.BeginTask(weight, "Reading header part 5"))
{
part5 = await this.ReadHeaderPart5Async(stream, toc.OffsetToPart5, NefsHeaderPart5.Size, p);
}
using (p.BeginTask(weight, "Reading header part 6"))
{
part6 = await this.Read20HeaderPart6Async(part6Stream, (uint)part6Offset + toc.OffsetToPart6, part1, p);
}
using (p.BeginTask(weight, "Reading header part 7"))
{
var numEntries = (uint)part2.EntriesByIndex.Count;
part7 = await this.ReadHeaderPart7Async(part6Stream, (uint)part6Offset + toc.OffsetToPart7, numEntries, p);
}
using (p.BeginTask(weight, "Reading header part 8"))
{
var part8Size = intro.HeaderSize - toc.OffsetToPart8;
part8 = await this.ReadHeaderPart8Async(stream, toc.OffsetToPart8, part8Size, p);
}
// Validate header hash
if (!this.ValidateHash(stream, offset, intro))
{
Log.LogWarning("Header hash does not match expected value.");
}
// The header stream must be disposed
stream.Dispose();
return(new Nefs20Header(intro, toc, part1, part2, part3, part4, part5, part6, part7, part8));
}
public async Task WriterHeaderIntroTocAsync_ValidData_Written()
{
var toc = new Nefs20HeaderIntroToc();
toc.Data0x04_OffsetToPart1.Value = 111;
toc.Data0x0c_OffsetToPart2.Value = 222;
toc.Data0x14_OffsetToPart3.Value = 333;
toc.Data0x18_OffsetToPart4.Value = 444;
toc.Data0x1c_OffsetToPart5.Value = 555;
toc.Data0x08_OffsetToPart6.Value = 666;
toc.Data0x10_OffsetToPart7.Value = 777;
toc.Data0x20_OffsetToPart8.Value = 888;
toc.Data0x00_NumVolumes.Value = 404;
toc.Data0x02_HashBlockSize.Value = 505;
// This chunk of data is unknown, but it must be 0x5C bytes long
toc.Data0x24_Unknown.Value = new byte[0x5C];
toc.Data0x24_Unknown.Value[0] = 20;
/*
* Write
*/
var writer = this.CreateWriter();
byte[] buffer;
var offset = 10;
using (var ms = new MemoryStream())
{
await writer.WriteHeaderIntroTocAsync(ms, (uint)offset, toc, new NefsProgress());
buffer = ms.ToArray();
}
/*
* Verify
*/
// Num volumes
Assert.Equal(404, BitConverter.ToInt16(buffer, offset + 0x00));
// Hash block size
Assert.Equal(505, BitConverter.ToInt16(buffer, offset + 0x02));
// Offset to part 1
Assert.Equal(111, BitConverter.ToInt32(buffer, offset + 0x04));
// Offset to part 6
Assert.Equal(666, BitConverter.ToInt32(buffer, offset + 0x08));
// Offset to part 2
Assert.Equal(222, BitConverter.ToInt32(buffer, offset + 0x0C));
// Offset to part 7
Assert.Equal(777, BitConverter.ToInt32(buffer, offset + 0x10));
// Offset to part 3
Assert.Equal(333, BitConverter.ToInt32(buffer, offset + 0x14));
// Offset to part 4
Assert.Equal(444, BitConverter.ToInt32(buffer, offset + 0x18));
// Offset to part 5
Assert.Equal(555, BitConverter.ToInt32(buffer, offset + 0x1C));
// Offset to part 8
Assert.Equal(888, BitConverter.ToInt32(buffer, offset + 0x20));
// 0x24 Unknown
Assert.Equal(20, buffer[offset + 0x24]);
}
/// <summary>
/// Writes an archive to the specified stream. A new archive obejct is returned that
/// contains the updated header and item metadata.
/// </summary>
/// <param name="stream">The stream to write to.</param>
/// <param name="sourceHeader">Donor header information.</param>
/// <param name="sourceItems">List of items to write. This list is not modified directly.</param>
/// <param name="workDir">Temp working directory path.</param>
/// <param name="p">Progress info.</param>
/// <returns>A new NefsArchive object containing the updated header and item metadata.</returns>
private async Task <NefsArchive> WriteArchiveAsync(
Stream stream,
Nefs20Header sourceHeader,
NefsItemList sourceItems,
string workDir,
NefsProgress p)
{
// Setup task weights
var taskWeightPrepareItems = 0.45f;
var taskWeightWriteItems = 0.45f;
var taskWeightHeader = 0.1f;
// Prepare items for writing
NefsItemList items;
using (var t = p.BeginTask(taskWeightPrepareItems, "Preparing items"))
{
items = await this.PrepareItemsAsync(sourceItems, workDir, p);
}
// Determine number of items
var numItems = items.Count;
// Update header parts 3 and 4 first (need to know their sizes)
var p4 = new Nefs20HeaderPart4(items);
var p3 = new NefsHeaderPart3(items);
// Compute header size
var introSize = NefsHeaderIntro.Size;
var tocSize = Nefs20HeaderIntroToc.Size;
var p1Size = numItems * NefsHeaderPart1Entry.Size; // TODO : What about duplicates?
var p2Size = numItems * NefsHeaderPart2Entry.Size; // TODO : What about duplicates?
var p3Size = p3.Size;
var p4Size = p4.Size;
var p5Size = NefsHeaderPart5.Size;
var p6Size = numItems * Nefs20HeaderPart6Entry.Size;
var p7Size = numItems * NefsHeaderPart7Entry.Size;
var p8Size = sourceHeader.Intro.HeaderSize - sourceHeader.TableOfContents.OffsetToPart8;
var headerSize = introSize + tocSize + p1Size + p2Size + p3Size + p4Size + p5Size + p6Size + p7Size + p8Size;
// Determine first data offset. There are two known offset values. If the header is
// large enough, the second (larger) offset is used.
var firstDataOffset = Nefs20Header.DataOffsetDefault;
if (headerSize > firstDataOffset)
{
firstDataOffset = Nefs20Header.DataOffsetLarge;
}
// Write item data
UInt64 archiveSize;
using (var t = p.BeginTask(taskWeightWriteItems, "Writing items"))
{
archiveSize = await this.WriteItemsAsync(stream, items, firstDataOffset, p);
}
// Update remaining header data
var p1 = new NefsHeaderPart1(items, p4);
var p2 = new NefsHeaderPart2(items, p3);
var p6 = new Nefs20HeaderPart6(items);
var p7 = new NefsHeaderPart7(items);
// Compute total archive size
var p5 = new NefsHeaderPart5();
p5.Data0x00_ArchiveSize.Value = archiveSize;
p5.Data0x08_ArchiveNameStringOffset.Value = p3.OffsetsByFileName[items.DataFileName];
p5.Data0x0C_FirstDataOffset.Value = sourceHeader.Part5.FirstDataOffset;
// Update header intro
var intro = new NefsHeaderIntro();
intro.Data0x00_MagicNumber.Value = sourceHeader.Intro.MagicNumber;
intro.Data0x24_AesKeyHexString.Value = sourceHeader.Intro.AesKeyHexString;
intro.Data0x64_HeaderSize.Value = (uint)headerSize;
intro.Data0x68_NefsVersion.Value = sourceHeader.Intro.NefsVersion;
intro.Data0x6c_NumberOfItems.Value = (uint)numItems;
intro.Data0x70_UnknownZlib.Value = sourceHeader.Intro.Unknown0x70zlib;
intro.Data0x78_Unknown.Value = sourceHeader.Intro.Unknown0x78;
var toc = new Nefs20HeaderIntroToc();
toc.Data0x00_NumVolumes.Value = sourceHeader.TableOfContents.NumVolumes;
toc.Data0x02_HashBlockSize.Value = sourceHeader.TableOfContents.Data0x02_HashBlockSize.Value;
toc.Data0x04_OffsetToPart1.Value = introSize + tocSize;
toc.Data0x0c_OffsetToPart2.Value = toc.OffsetToPart1 + (uint)p1Size;
toc.Data0x14_OffsetToPart3.Value = toc.OffsetToPart2 + (uint)p2Size;
toc.Data0x18_OffsetToPart4.Value = toc.OffsetToPart3 + (uint)p3Size;
toc.Data0x1c_OffsetToPart5.Value = toc.OffsetToPart4 + (uint)p4Size;
toc.Data0x08_OffsetToPart6.Value = toc.OffsetToPart5 + (uint)p5Size;
toc.Data0x10_OffsetToPart7.Value = toc.OffsetToPart6 + (uint)p6Size;
toc.Data0x20_OffsetToPart8.Value = toc.OffsetToPart7 + (uint)p7Size;
toc.Data0x24_Unknown.Value = sourceHeader.TableOfContents.Unknown0x24;
// Part 8 - not writing anything for now
var p8 = new NefsHeaderPart8(p8Size);
// Create new header object
var header = new Nefs20Header(intro, toc, p1, p2, p3, p4, p5, p6, p7, p8);
// Write the header
using (var t = p.BeginTask(taskWeightHeader, "Writing header"))
{
await this.WriteHeaderAsync(stream, 0, header, p);
}
// Update hash
await this.UpdateHashAsync(stream, 0, header, p);
// Create new archive object
return(new NefsArchive(header, items));
}