/// <summary>
/// Creates an item from header data.
/// </summary>
/// <param name="id">The item id.</param>
/// <param name="header">The header data.</param>
/// <param name="dataSourceList">The item list to use as the item data source.</param>
/// <returns>A new <see cref="NefsItem"/>.</returns>
public static NefsItem CreateFromHeader(NefsItemId id, NefsHeader header, NefsItemList dataSourceList)
{
var p1 = header.Part1.EntriesById[id];
var p2 = header.Part2.EntriesById[id];
// Check if part 6 exists
NefsHeaderPart6Entry p6 = null;
if (header.Part6.EntriesById.ContainsKey(id))
{
p6 = header.Part6.EntriesById[id];
}
// Determine type
var type = p2.Data0x0c_ExtractedSize.Value == 0 ? NefsItemType.Directory : NefsItemType.File;
// Find parent
var parentId = header.GetItemDirectoryId(id);
// Offset and size
var dataOffset = p1.Data0x00_OffsetToData.Value;
var extractedSize = p2.Data0x0c_ExtractedSize.Value;
// Data source
INefsDataSource dataSource;
if (type == NefsItemType.Directory)
{
// Item is a directory
dataSource = new NefsEmptyDataSource();
}
else if (p1.IndexIntoPart4 == 0xFFFFFFFFU)
{
// Item is not compressed
var size = new NefsItemSize(extractedSize);
dataSource = new NefsItemListDataSource(dataSourceList, dataOffset, size);
}
else
{
// Item is compressed
var p4 = header.Part4.EntriesByIndex[p1.IndexIntoPart4];
var size = new NefsItemSize(extractedSize, p4.ChunkSizes);
dataSource = new NefsItemListDataSource(dataSourceList, dataOffset, size);
}
// File name and path
var fileName = header.GetItemFileName(id);
// Gather unknown metadata
var unknown = new NefsItemUnknownData
{
Part6Unknown0x00 = p6?.Byte0 ?? 0,
Part6Unknown0x01 = p6?.Byte1 ?? 0,
Part6Unknown0x02 = p6?.Byte2 ?? 0,
Part6Unknown0x03 = p6?.Byte3 ?? 0,
};
// Create item
return(new NefsItem(id, fileName, parentId, type, dataSource, unknown));
}
/// <inheritdoc/>
public async Task <NefsArchive> ReadArchiveAsync(
string headerFilePath,
ulong headerOffset,
string dataFilePath,
NefsProgress p)
{
// Validate header path
if (!this.FileSystem.File.Exists(headerFilePath))
{
throw new FileNotFoundException($"File not found: {headerFilePath}.");
}
// Validate data path
if (!this.FileSystem.File.Exists(dataFilePath))
{
throw new FileNotFoundException($"File not found: {dataFilePath}.");
}
// Read the header
NefsHeader header = null;
using (var stream = this.FileSystem.File.OpenRead(headerFilePath))
{
header = await this.ReadHeaderAsync(stream, headerOffset, p);
}
// Create items from header
var items = this.CreateItems(dataFilePath, header);
// Create the archive
return(new NefsArchive(header, items));
}
/// <summary>
/// Gets the new expected hash and writes it to the header.
/// </summary>
/// <param name="stream">The stream containing the header.</param>
/// <param name="headerOffset">The offset to the header in the stream.</param>
/// <param name="header">The header.</param>
/// <param name="p">Progress info.</param>
/// <returns>The async task.</returns>
private async Task UpdateHashAsync(
Stream stream,
UInt64 headerOffset,
NefsHeader header,
NefsProgress p)
{
// The hash is of the entire header expect for the expected hash
var firstOffset = (long)headerOffset;
var secondOffset = firstOffset + 0x24;
var headerSize = (int)header.Intro.HeaderSize;
// Seek to beginning of header
stream.Seek(firstOffset, SeekOrigin.Begin);
// Read magic num
var dataToHash = new byte[headerSize - 0x20];
await stream.ReadAsync(dataToHash, 0, 4);
// Skip expected hash and read rest of header
stream.Seek(secondOffset, SeekOrigin.Begin);
stream.Read(dataToHash, 4, headerSize - 0x24);
// Compute the new expected hash
using (var hash = SHA256.Create())
{
byte[] hashOut = hash.ComputeHash(dataToHash);
// Write the expected hash
header.Intro.Data0x04_ExpectedHash.Value = hashOut;
await header.Intro.Data0x04_ExpectedHash.WriteAsync(stream, headerOffset, p);
}
}
public NefsArchive(string filePath, NefsProgressInfo p)
{
float taskWeightHeader = 0.55f;
float taskWeightItems = 0.45f;
_filePath = Path.GetFullPath(filePath);
using (var file = new FileStream(_filePath, FileMode.Open))
{
/* Create a hash of this archive's file path */
_filePathHash = FilePathHelper.HashStringMD5(_filePath);
/******************************************************************
* READ HEADER
******************************************************************/
p.BeginTask(taskWeightHeader, "Reading NeFS header...");
_header = new NefsHeader(file, this, p);
if (!checkHash(file))
{
log.Error("Header hash does not match expected value.");
}
p.EndTask();
/******************************************************************
* READ ITEMS
******************************************************************/
p.BeginTask(taskWeightItems, "Loading NeFS items...");
var numEntries = _header.Part1.Entries.Count;
for (int i = 0; i < numEntries; i++)
{
p.BeginTask(1 / (float)numEntries, String.Format("Loading item {0}/{1}", i + 1, numEntries));
var entry = _header.Part1.Entries[i];
try
{
var item = new NefsItem(file, this, entry.Id);
_items.Add(item);
}
catch (Exception ex)
{
log.Warn("Error loading item with id " + entry.Id, ex);
}
p.EndTask();
}
p.EndTask();
}
}
/// <summary>
/// Creates a test archive. Does not write an archive to disk. Just creates a <see
/// cref="NefsArchive"/> object.
/// </summary>
/// <param name="filePath">The file path to use for the archive.</param>
/// <returns>A <see cref="NefsArchive"/>.</returns>
public static NefsArchive Create(string filePath)
{
var items = new NefsItemList(filePath);
Assert.Empty(items.EnumerateById());
Assert.Empty(items.EnumerateDepthFirstByName());
var intro = new NefsHeaderIntro();
intro.Data0x6c_NumberOfItems.Value = (uint)items.Count;
var toc = new NefsHeaderIntroToc();
var header = new NefsHeader(intro, toc, items);
return(new NefsArchive(header, items));
}
private NefsItemList CreateItems(string dataFilePath, NefsHeader h)
{
var items = new NefsItemList(dataFilePath);
var numItems = h.TableOfContents.Part1Size / NefsHeaderPart1Entry.Size;
for (var i = 0; i < numItems; ++i)
{
// Create the item
var id = new NefsItemId((uint)i);
try
{
var item = NefsItem.CreateFromHeader(id, h, items);
items.Add(item);
}
catch (Exception)
{
Log.LogError($"Failed to create item {id}, skipping.");
}
}
return(items);
}
/// <summary>
/// Creates a test archive. Does not write an archive to disk. Just creates a <see
/// cref="NefsArchive"/> object.
/// </summary>
/// <param name="filePath">The file path to use for the archive.</param>
/// <returns>A <see cref="NefsArchive"/>.</returns>
public static NefsArchive Create(string filePath)
{
var items = new NefsItemList(filePath);
var file1DataSource = new NefsItemListDataSource(items, File1Offset, new NefsItemSize(File1ExtractedSize, File1ChunkSizes));
var file1 = new NefsItem(new NefsItemId(File1ItemId), File1Name, new NefsItemId(File1DirectoryId), NefsItemType.File, file1DataSource, TestHelpers.CreateUnknownData());
items.Add(file1);
var dir1DataSource = new NefsEmptyDataSource();
var dir1 = new NefsItem(new NefsItemId(Dir1ItemId), Dir1Name, new NefsItemId(Dir1DirectoryId), NefsItemType.Directory, dir1DataSource, TestHelpers.CreateUnknownData());
items.Add(dir1);
var file2DataSource = new NefsItemListDataSource(items, File2Offset, new NefsItemSize(File2ExtractedSize, File2ChunkSizes));
var file2 = new NefsItem(new NefsItemId(File2ItemId), File2Name, new NefsItemId(File2DirectoryId), NefsItemType.File, file2DataSource, TestHelpers.CreateUnknownData());
items.Add(file2);
var file3DataSource = new NefsItemListDataSource(items, File3Offset, new NefsItemSize(File3ExtractedSize, File3ChunkSizes));
var file3 = new NefsItem(new NefsItemId(File3ItemId), File3Name, new NefsItemId(File3DirectoryId), NefsItemType.File, file3DataSource, TestHelpers.CreateUnknownData());
items.Add(file3);
Assert.Equal((int)NumItems, items.Count);
var intro = new NefsHeaderIntro();
intro.Data0x6c_NumberOfItems.Value = (uint)items.Count;
var toc = new NefsHeaderIntroToc();
var header = new NefsHeader(intro, toc, items);
return(new NefsArchive(header, items));
}
/// <summary>
/// Initializes a new instance of the <see cref="NefsArchive"/> class.
/// </summary>
/// <param name="header">The archive's header.</param>
/// <param name="items">List of items for this archive.</param>
public NefsArchive(NefsHeader header, NefsItemList items)
{
this.Header = header ?? throw new ArgumentNullException(nameof(header));
this.Items = items ?? throw new ArgumentNullException(nameof(items));
}
/// <summary>
/// Writes the header to the output stream.
/// </summary>
/// <param name="stream">The stream to write to.</param>
/// <param name="headerOffset">The offset into the stream to begin.</param>
/// <param name="header">The header to write.</param>
/// <param name="p">Progress info.</param>
/// <returns>The async task.</returns>
private async Task WriteHeaderAsync(Stream stream, UInt64 headerOffset, NefsHeader header, NefsProgress p)
{
// Calc weight of each task (8 parts + intro + table of contents)
var weight = 1.0f / 10.0f;
// Get table of contents
var toc = header.TableOfContents;
using (var t = p.BeginTask(weight, "Writing header intro"))
{
var offset = headerOffset + NefsHeader.IntroOffset;
await this.WriteHeaderIntroAsync(stream, offset, header.Intro, p);
}
using (var t = p.BeginTask(weight, "Writing header intro table of contents"))
{
var offset = headerOffset + NefsHeaderIntroToc.Offset;
await this.WriteHeaderIntroTocAsync(stream, offset, header.TableOfContents, p);
}
using (var t = p.BeginTask(weight, "Writing header part 1"))
{
var offset = headerOffset + toc.OffsetToPart1;
await this.WriteHeaderPart1Async(stream, offset, header.Part1, p);
}
using (var t = p.BeginTask(weight, "Writing header part 2"))
{
var offset = headerOffset + toc.OffsetToPart2;
await this.WriteHeaderPart2Async(stream, offset, header.Part2, p);
}
using (var t = p.BeginTask(weight, "Writing header part 3"))
{
var offset = headerOffset + toc.OffsetToPart3;
await this.WriteHeaderPart3Async(stream, offset, header.Part3, p);
}
using (var t = p.BeginTask(weight, "Writing header part 4"))
{
var offset = headerOffset + toc.OffsetToPart4;
await this.WriteHeaderPart4Async(stream, offset, header.Part4, p);
}
using (var t = p.BeginTask(weight, "Writing header part 5"))
{
var offset = headerOffset + toc.OffsetToPart5;
await this.WriteHeaderPart5Async(stream, offset, header.Part5, p);
}
using (var t = p.BeginTask(weight, "Writing header part 6"))
{
var offset = headerOffset + toc.OffsetToPart6;
await this.WriteHeaderPart6Async(stream, offset, header.Part6, p);
}
using (var t = p.BeginTask(weight, "Writing header part 7"))
{
var offset = headerOffset + toc.OffsetToPart7;
await this.WriteHeaderPart7Async(stream, offset, header.Part7, p);
}
using (var t = p.BeginTask(weight, "Writing header part 8"))
{
var offset = headerOffset + toc.OffsetToPart8;
await this.WriteHeaderPart8Async(stream, offset, header.Part8, p);
}
}
/// <summary>
/// Writes an archive to the specified stream. A new archive obejct is returned that
/// contains the updated header and item metadata.
/// </summary>
/// <param name="stream">The stream to write to.</param>
/// <param name="sourceHeader">Donor header information.</param>
/// <param name="sourceItems">List of items to write. This list is not modified directly.</param>
/// <param name="workDir">Temp working directory path.</param>
/// <param name="p">Progress info.</param>
/// <returns>A new NefsArchive object containing the updated header and item metadata.</returns>
private async Task <NefsArchive> WriteArchiveAsync(
Stream stream,
NefsHeader sourceHeader,
NefsItemList sourceItems,
string workDir,
NefsProgress p)
{
// Setup task weights
var taskWeightPrepareItems = 0.45f;
var taskWeightWriteItems = 0.45f;
var taskWeightHeader = 0.1f;
// Prepare items for writing
NefsItemList items;
using (var t = p.BeginTask(taskWeightPrepareItems, "Preparing items"))
{
items = await this.PrepareItemsAsync(sourceItems, workDir, NefsHeader.ChunkSize, p);
}
// Determine number of items
var numItems = items.Count;
// Update header parts 3 and 4 first (need to know their sizes)
var p4 = new NefsHeaderPart4(items);
var p3 = new NefsHeaderPart3(items);
// Compute header size
var introSize = NefsHeaderIntro.Size;
var tocSize = NefsHeaderIntroToc.Size;
var p1Size = numItems * NefsHeaderPart1Entry.Size;
var p2Size = numItems * NefsHeaderPart2Entry.Size;
var p3Size = p3.Size;
var p4Size = p4.Size;
var p5Size = NefsHeaderPart5.Size;
var p6Size = numItems * NefsHeaderPart6Entry.Size;
var p7Size = numItems * NefsHeaderPart7Entry.Size;
var p8Size = sourceHeader.Intro.HeaderSize - sourceHeader.TableOfContents.OffsetToPart8;
var headerSize = introSize + tocSize + p1Size + p2Size + p3Size + p4Size + p5Size + p6Size + p7Size + p8Size;
// Determine first data offset. There are two known offset values. If the header is
// large enough, the second (larger) offset is used.
var firstDataOffset = NefsHeader.DataOffsetDefault;
if (headerSize > firstDataOffset)
{
firstDataOffset = NefsHeader.DataOffsetLarge;
}
// Write item data
UInt64 archiveSize;
using (var t = p.BeginTask(taskWeightWriteItems, "Writing items"))
{
archiveSize = await this.WriteItemsAsync(stream, items, firstDataOffset, p);
}
// Update remaining header data
var p1 = new NefsHeaderPart1(items, p4);
var p2 = new NefsHeaderPart2(items, p3);
var p6 = new NefsHeaderPart6(items);
var p7 = new NefsHeaderPart7(items);
// Compute total archive size
var p5 = new NefsHeaderPart5();
p5.Data0x00_ArchiveSize.Value = archiveSize;
p5.Data0x08_ArchiveNameStringOffset.Value = p3.OffsetsByFileName[items.DataFileName];
p5.Data0x0C_FirstDataOffset.Value = sourceHeader.Part5.FirstDataOffset;
// Update header intro
var intro = new NefsHeaderIntro();
intro.Data0x00_MagicNumber.Value = sourceHeader.Intro.MagicNumber;
intro.Data0x24_AesKeyHexString.Value = sourceHeader.Intro.AesKeyHexString;
intro.Data0x64_HeaderSize.Value = (uint)headerSize;
intro.Data0x68_Unknown.Value = sourceHeader.Intro.Unknown0x68;
intro.Data0x6c_NumberOfItems.Value = (uint)numItems;
intro.Data0x70_UnknownZlib.Value = sourceHeader.Intro.Unknown0x70zlib;
intro.Data0x78_Unknown.Value = sourceHeader.Intro.Unknown0x78;
var toc = new NefsHeaderIntroToc();
toc.Data0x00_Unknown.Value = sourceHeader.TableOfContents.Unknown0x00;
toc.Data0x04_OffsetToPart1.Value = introSize + tocSize;
toc.Data0x0c_OffsetToPart2.Value = toc.OffsetToPart1 + (uint)p1Size;
toc.Data0x14_OffsetToPart3.Value = toc.OffsetToPart2 + (uint)p2Size;
toc.Data0x18_OffsetToPart4.Value = toc.OffsetToPart3 + (uint)p3Size;
toc.Data0x1c_OffsetToPart5.Value = toc.OffsetToPart4 + (uint)p4Size;
toc.Data0x08_OffsetToPart6.Value = toc.OffsetToPart5 + (uint)p5Size;
toc.Data0x10_OffsetToPart7.Value = toc.OffsetToPart6 + (uint)p6Size;
toc.Data0x20_OffsetToPart8.Value = toc.OffsetToPart7 + (uint)p7Size;
toc.Data0x24_Unknown.Value = sourceHeader.TableOfContents.Unknown0x24;
// Part 8 - not writing anything for now
var p8 = new NefsHeaderPart8((uint)p8Size);
// Create new header object
var header = new NefsHeader(intro, toc, p1, p2, p3, p4, p5, p6, p7, p8);
// Write the header
using (var t = p.BeginTask(taskWeightHeader, "Writing header"))
{
await this.WriteHeaderAsync(stream, 0, header, p);
}
// Update hash
await this.UpdateHashAsync(stream, 0, header, p);
// Create new archive object
return(new NefsArchive(header, items));
}
