public void Write(BinaryWriter bw, IndexFooter footer)
{
bw.Write(Key.Value, 0, footer.KeySize);
bw.WriteUIntBE(CompressedSize, footer.CompressedSizeBytes);
if (footer.OffsetBytes == 6)
{
bw.WriteUInt16BE(IndexOrdinal);
}
if (footer.OffsetBytes >= 4)
{
bw.WriteUInt32BE(Offset);
}
}
/// <summary>
/// Creates a new IndexFile
/// </summary>
public IndexFile(IndexType type)
{
_indexEntries = new SortedList <MD5Hash, IndexEntry>(new MD5HashComparer());
_newEntries = new Dictionary <MD5Hash, CASRecord>();
Type = type;
IndexFooter = new IndexFooter();
if (IsLooseIndex)
{
IndexFooter.OffsetBytes = 0;
}
else if (IsGroupIndex)
{
IndexFooter.OffsetBytes = 6;
}
}
private void Read(Stream stream)
{
if (!stream.CanRead || stream.Length <= 0)
{
throw new NotSupportedException($"Unable to read IndexFile stream");
}
using (var md5 = MD5.Create())
using (var reader = new BinaryReader(stream))
{
IndexFooter.Read(reader);
// calculate file dimensions
var(PageSize, _, PageCount) = GetFileDimensions();
// read the entries
_indexEntries.Capacity = (int)IndexFooter.EntryCount;
stream.Position = 0;
for (int i = 0; i < PageCount; i++)
{
// buffer the page then read the entries to minimise file reads
using (var ms = new MemoryStream(reader.ReadBytes(PageSize)))
using (var br = new BinaryReader(ms))
{
var entry = new IndexEntry();
while (entry.Read(br, IndexFooter))
{
_indexEntries[entry.Key] = entry;
entry = new IndexEntry();
}
}
}
// calculate the current filename
Checksum = stream.HashSlice(md5, stream.Length - IndexFooter.Size + IndexFooter.ChecksumSize, IndexFooter.Size - IndexFooter.ChecksumSize);
// store the current blob offset
_currentOffset = (ulong)_indexEntries.Values.Sum(x => (long)x.CompressedSize);
}
}
public bool Read(BinaryReader br, IndexFooter footer)
{
Key = new MD5Hash(br.ReadBytes(footer.KeySize));
if (Key.IsEmpty)
{
return(false);
}
CompressedSize = br.ReadUIntBE(footer.CompressedSizeBytes);
if (footer.OffsetBytes == 6)
{
IndexOrdinal = br.ReadUInt16BE();
}
if (footer.OffsetBytes >= 4)
{
Offset = br.ReadUInt32BE();
}
return(true);
}
/// <summary>
/// Saves the IndexFile to disk and optionally updates the CDN config
/// </summary>
/// <param name="directory"></param>
/// <param name="configContainer"></param>
public void Write(string directory, Configs.ConfigContainer configContainer = null)
{
RequiresSave = false;
// TODO patch index writing
if (IsPatchIndex || Type == IndexType.Unknown)
{
throw new NotImplementedException();
}
// Group Indicies only supported for Data and Patch indicies
if (IsGroupIndex && IsLooseIndex)
{
throw new NotImplementedException();
}
List <MD5Hash> EKeyLookupHashes = new List <MD5Hash>();
List <MD5Hash> PageChecksums = new List <MD5Hash>();
// update Footer
IndexFooter.EntryCount = (uint)_indexEntries.Count;
// get file dimensions
var(PageSize, EntriesPerPage, PageCount) = GetFileDimensions();
using var md5 = MD5.Create();
using var ms = new MemoryStream(PageCount * (PageSize + 1));
using var bw = new BinaryWriter(ms);
// set capcity
EKeyLookupHashes.Capacity = PageCount;
PageChecksums.Capacity = PageCount;
// IndexEntries
int index = 0;
for (int i = 0; i < PageCount; i++)
{
// write the entries
for (int j = 0; j < EntriesPerPage && index < IndexFooter.EntryCount; j++)
{
_indexEntries.Values[index++].Write(bw, IndexFooter);
}
// apply padding and store EKey and page checksum
int remainder = (int)bw.BaseStream.Position % PageSize;
if (remainder > 0)
{
ms.Write(new byte[PageSize - remainder]);
}
EKeyLookupHashes.Add(_indexEntries.Values[index - 1].Key);
PageChecksums.Add(ms.HashSlice(md5, bw.BaseStream.Position - PageSize, PageSize, IndexFooter.ChecksumSize));
}
// EKey Lookup
long lookupStartPos = bw.BaseStream.Position;
foreach (var lookupHash in EKeyLookupHashes)
{
bw.Write(lookupHash.Value);
}
// Page hashes - final page is ignored
long pageStartPos = bw.BaseStream.Position;
PageChecksums.RemoveAt(PageChecksums.Count - 1);
foreach (var pagechecksum in PageChecksums)
{
bw.Write(pagechecksum.Value);
}
// LastPage hash - last PageSize of Entries
long footerStartPos = bw.BaseStream.Position;
IndexFooter.LastPageHash = ms.HashSlice(md5, lookupStartPos - PageSize, PageSize, IndexFooter.ChecksumSize);
bw.Write(IndexFooter.LastPageHash.Value);
// TOC hash - from EKey Lookup to LastPage Hash
IndexFooter.ContentsHash = ms.HashSlice(md5, lookupStartPos, ms.Length - lookupStartPos, IndexFooter.ChecksumSize);
bw.Write(IndexFooter.ContentsHash.Value);
// write footer
IndexFooter.Write(bw);
// compute filename - from ContentsHash to EOF
MD5Hash newChecksum = ms.HashSlice(md5, footerStartPos + IndexFooter.ChecksumSize, IndexFooter.Size - IndexFooter.ChecksumSize);
// update the CDN Config
UpdateConfig(configContainer, newChecksum, bw.BaseStream.Length);
//// remove old index file
//if (!Checksum.IsEmpty)
// Helpers.Delete(Checksum.ToString() + ".index", directory);
// update Checksum
Checksum = newChecksum;
// Group Indicies are generated client-side
if (IsGroupIndex)
{
return;
}
string saveLocation = Helpers.GetCDNPath(Checksum.ToString() + ".index", "data", directory, true);
if (!File.Exists(saveLocation))
{
// save to disk
File.WriteAllBytes(saveLocation, ms.ToArray());
}
}
