/// <summary>
/// Parses all Index files in the provided directory.
/// <para></para>
/// Providing the ConfigContainer will filter indices to just those used in the CDNConfig.
/// </summary>
/// <param name="directory">Directory the archives are located</param>
/// <param name="configContainer">The Configs for the repo</param>
/// <param name="useParallelism">Enables parallel processing</param>
public void Open(string directory, Configs.ConfigContainer configContainer = null, bool useParallelism = false)
{
IsRemote = false;
_sourceDirectory = directory;
_useParallelism = useParallelism;
if (!Directory.Exists(directory))
{
throw new ArgumentException("Directory not found", paramName: nameof(directory));
}
var indices = Directory.EnumerateFiles(directory, "*.index", SearchOption.AllDirectories);
// filter the indices to just this version's
if (configContainer != null)
{
var applicableIndicies = GetRequiredIndices(configContainer);
indices = indices.Where(x => applicableIndicies.Contains(Path.GetFileNameWithoutExtension(x)));
}
ParallelOptions options = new ParallelOptions()
{
MaxDegreeOfParallelism = useParallelism ? -1 : 1
};
Parallel.ForEach(indices, options, index => _indices.Add(new IndexFile(index)));
}
/// <summary>
/// Downloads all Index and Archive files from a remote CDN
/// </summary>
/// <param name="directory"></param>
/// <param name="configContainer"></param>
public void DownloadRemote(string directory, Configs.ConfigContainer configContainer, Configs.ManifestContainer manifestContainer)
{
_client = new CDNClient(manifestContainer);
var queuedDownloader = new QueuedDownloader(directory, _client);
// download data archives
var archives = configContainer.CDNConfig.GetValues("archives");
if (archives != null && archives.Count > 0)
{
queuedDownloader.Enqueue(archives);
queuedDownloader.Enqueue(archives, (x) => x + ".index");
queuedDownloader.Download("data");
}
// download patch archives
var patcharchives = configContainer.CDNConfig.GetValues("patch-archives");
if (patcharchives != null && patcharchives.Count > 0)
{
queuedDownloader.Enqueue(patcharchives);
queuedDownloader.Enqueue(patcharchives, (x) => x + ".index");
queuedDownloader.Download("patch");
}
// download loose file index
var fileIndex = configContainer.CDNConfig.GetValue("file-index");
if (fileIndex != null)
{
string url = Helpers.GetCDNUrl(fileIndex, "data");
string path = Helpers.GetCDNPath(fileIndex, "data", directory, true);
_client.DownloadFile(url, path).Wait();
// download loose files
var index = new IndexFile(path);
queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString());
queuedDownloader.Download("data");
}
// download loose patch file index
var patchIndex = configContainer.CDNConfig.GetValue("patch-file-index");
if (patchIndex != null)
{
string url = Helpers.GetCDNUrl(patchIndex, "patch");
string path = Helpers.GetCDNPath(patchIndex, "patch", directory, true);
_client.DownloadFile(url, path).Wait();
// download loose patches
var index = new IndexFile(path);
queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString());
queuedDownloader.Download("patch");
}
Open(directory);
}
private void UpdateConfig(Configs.ConfigContainer configContainer, MD5Hash hash, long size)
{
if (configContainer?.CDNConfig == null)
{
return;
}
// determine the field names
string archivefield, sizefield;
if (IsGroupIndex)
{
archivefield = IsPatchIndex ? "patch-archive-group" : "archive-group";
sizefield = "";
}
else if (IsLooseIndex)
{
archivefield = IsPatchIndex ? "patch-file-index" : "file-index";
sizefield = archivefield + "-size";
}
else
{
archivefield = IsPatchIndex ? "patch-archives" : "archives";
sizefield = archivefield + "-index-size";
}
// update the collections
var archives = configContainer.CDNConfig.GetValues(archivefield);
var sizes = configContainer.CDNConfig.GetValues(sizefield);
if (archives != null)
{
if (IsGroupIndex)
{
archives[0] = hash.ToString(); // group indicies are single entries
}
else
{
// remove old hash
if (Checksum.Value != null)
{
int index = archives.IndexOf(Checksum.ToString());
if (index > -1)
{
archives.RemoveAt(index);
sizes?.RemoveAt(index);
}
}
// add if new
if (!archives.Contains(hash.ToString()))
{
archives.Add(hash.ToString());
sizes?.Add(size.ToString());
}
}
}
}
/// <summary>
/// Updates modified data indices and writes enqueued files to archives
/// <para>Note: IndexFile saving is limited to new entries if the container was opened remotely</para>
/// </summary>
/// <param name="directory"></param>
/// <param name="dispose">Delete old files</param>
/// <param name="configContainer"></param>
public void Save(string directory, Configs.ConfigContainer configContainer = null)
{
bool sameDirectory = directory.EqualsIC(_sourceDirectory);
// save altered Data archive indices
if (!IsRemote)
{
foreach (var index in DataIndices)
{
if (index.IsGroupIndex)
{
continue;
}
if (index.RequiresSave)
{
// save the index file and blob
string prevBlob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
index.Write(directory, configContainer);
index.WriteBlob(directory, prevBlob);
}
else if (!sameDirectory)
{
// copy the index file and blob
string oldblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
string newblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", directory, true);
File.Copy(oldblob, newblob);
File.Copy(oldblob + ".index", newblob + ".index");
}
}
}
// prevent duplicated entries
var duplicates = QueuedEntries.Keys
.Where(k => GetIndexFileAndEntry(IndexType.Data, k, out _) != null)
.ToArray();
foreach (var key in duplicates)
{
QueuedEntries.Remove(key);
}
// create any new archive indices
var partitions = EnumerablePartitioner.ConcreteBatch(QueuedEntries.Values, ArchiveDataSize, (x) => x.EBlock.CompressedSize);
foreach (var entries in partitions)
{
IndexFile index = new IndexFile(IndexType.Data);
index.Add(entries);
index.Write(directory, configContainer);
index.WriteBlob(directory);
}
// reload indices
Open(directory, useParallelism: _useParallelism);
}
/// <summary>
/// Saves the EncodingFile to disk and optionally updates the BuildConfig
/// </summary>
/// <param name="directory">Root Directory</param>
/// <param name="configContainer"></param>
/// <returns></returns>
public CASRecord Write(string directory, Configs.ConfigContainer configContainer = null)
{
if (Partial)
{
throw new NotSupportedException("Writing is not supported for partial EncodingFiles");
}
EBlock[] eblocks = new EBlock[_EncodingMap.Length];
CASRecord record;
using (var bt = new BlockTableStreamWriter(_EncodingMap[1], 1))
using (var bw = new BinaryWriter(bt))
{
// ESpecStringTable 1
bt.Write(string.Join('\0', ESpecStringTable).GetBytes());
EncodingHeader.ESpecTableSize = (uint)bt.Length;
// CKeysPageIndices 2, CKeysPageTable 3
WritePage(bw, eblocks, 2, EncodingHeader.CKeyPageSize << 10, _CKeyEntries);
// EKeysPageIndices 4, EKeysPageTable 5
WritePage(bw, eblocks, 4, EncodingHeader.EKeyPageSize << 10, _EKeyEntries);
// Header 0
bt.AddBlock(_EncodingMap[0], 0);
EncodingHeader.Write(bw);
// File ESpec 6
bt.AddBlock(_EncodingMap[6], 6);
bt.Write(GetFileESpec(bt.SubStreams).GetBytes());
// finalise
record = bt.Finalise();
// save
string saveLocation = Helpers.GetCDNPath(record.EKey.ToString(), "data", directory, true);
using (var fs = File.Create(saveLocation))
{
bt.WriteTo(fs);
record.BLTEPath = saveLocation;
}
}
// update the build config with the new values
if (configContainer?.BuildConfig != null)
{
configContainer.BuildConfig.SetValue("encoding-size", record.EBlock.DecompressedSize, 0);
configContainer.BuildConfig.SetValue("encoding-size", record.EBlock.CompressedSize, 1);
configContainer.BuildConfig.SetValue("encoding", record.CKey, 0);
configContainer.BuildConfig.SetValue("encoding", record.EKey, 1);
}
Checksum = record.CKey;
return(record);
}
/// <summary>
/// Updates modified data indices and writes enqueued files to archives
/// </summary>
/// <param name="directory"></param>
/// <param name="dispose">Delete old files</param>
/// <param name="configContainer"></param>
public void Save(string directory, Configs.ConfigContainer configContainer = null)
{
bool sameDirectory = directory.Equals(_sourceDirectory, StringComparison.OrdinalIgnoreCase);
// save altered Data archive indices
foreach (var index in DataIndices)
{
if (index.IsGroupIndex)
{
continue;
}
if (index.RequiresSave)
{
// save the index file and blob
string prevBlob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
index.Write(directory, configContainer);
index.WriteBlob(directory, prevBlob);
}
else if (!sameDirectory)
{
// copy the index file and blob
string oldblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
string newblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", directory, true);
File.Copy(oldblob, newblob);
File.Copy(oldblob + ".index", newblob + ".index");
}
}
// create any new archive indices
var partitions = EnumerablePartitioner.ConcreteBatch(_fileQueue.Values, ArchiveDataSize, (x) => x.EBlock.CompressedSize);
foreach (var entries in partitions)
{
IndexFile index = new IndexFile(IndexType.Data);
index.Add(entries);
index.Write(directory, configContainer);
index.WriteBlob(directory);
}
// TODO 1. verify if this is required 2. fix
// compute the Data Index Group hash
//GenerateIndexGroup(directory, configContainer);
// reload indices
_indices.Clear();
Open(directory, _useParallelism);
}
/// <summary>
/// Parses all Index files from a remote CDN
/// </summary>
/// <param name="manifestContainer"></param>
/// <param name="useParallelism"></param>
public void OpenRemote(Configs.ConfigContainer configContainer, Configs.ManifestContainer manifestContainer, bool useParallelism = false)
{
IsRemote = true;
_indices.Clear();
_useParallelism = useParallelism;
_client = new CDNClient(manifestContainer);
ParallelOptions options = new ParallelOptions()
{
MaxDegreeOfParallelism = useParallelism ? -1 : 1
};
// stream data archive indicies
var archives = configContainer.CDNConfig.GetValues("archives");
if (archives != null && archives.Count > 0)
{
Parallel.ForEach(archives, options, index => _indices.Add(new IndexFile(_client, index, IndexType.Data)));
}
// stream patch archive indices
var patcharchives = configContainer.CDNConfig.GetValues("patch-archives");
if (patcharchives != null && patcharchives.Count > 0)
{
Parallel.ForEach(patcharchives, options, index => _indices.Add(new IndexFile(_client, index, IndexType.Patch)));
}
// stream loose file index
var fileIndex = configContainer.CDNConfig.GetValue("file-index");
if (fileIndex != null)
{
_indices.Add(new IndexFile(_client, fileIndex, IndexType.Loose | IndexType.Data));
}
// stream loose patch file index
var patchIndex = configContainer.CDNConfig.GetValue("patch-file-index");
if (patchIndex != null)
{
_indices.Add(new IndexFile(_client, patchIndex, IndexType.Loose | IndexType.Patch));
}
}
/// <summary>
/// Creates a new CDN Client and loads the hosts from the CDNs file
/// </summary>
/// <param name="configContainer"></param>
/// <param name="applyDecryption"></param>
public CDNClient(Configs.ConfigContainer configContainer, bool applyDecryption = false) : this(applyDecryption)
{
if (configContainer.CDNsFile == null)
{
throw new ArgumentException("Unable to load CDNs file");
}
string[] hosts = configContainer.CDNsFile.GetValue("Hosts", configContainer.Locale)?.Split(' ');
foreach (var host in hosts)
{
Hosts.Add(host.Split('?')[0]);
}
if (Hosts.Count == 0)
{
throw new FormatException("No hosts found");
}
}
/// <summary>
/// Creates a fake Data Index Group and stores the computed checksum
/// </summary>
/// <param name="directory"></param>
/// <param name="configContainer"></param>
#pragma warning disable IDE0051 // Remove unused private members
private void GenerateIndexGroup(string directory, Configs.ConfigContainer configContainer)
#pragma warning restore IDE0051 // Remove unused private members
{
if (configContainer == null)
{
return;
}
// get the list of data archives
var archives = configContainer.CDNConfig.GetValues("archives");
archives.Sort(new MD5HashComparer());
// populate the archive indicies and
var temp = new List <IndexEntry>(DataIndices.Sum(x => x.Entries.Count()));
foreach (var index in DataIndices)
{
if (index.IsLooseIndex)
{
continue;
}
ushort archiveIndex = (ushort)archives.IndexOf(index.Checksum.ToString());
foreach (var e in index.Entries)
{
e.IndexOrdinal = archiveIndex;
temp.Add(e);
}
}
// sort
var comparer = new MD5HashComparer();
temp.Sort((x, y) => comparer.Compare(x.Key, y.Key));
// create a new IndexFile, add all entries and store the checksum in the CDN config
var indexFile = new IndexFile(IndexType.Data | IndexType.Group);
indexFile.LoadIndicies(temp);
indexFile.Write(directory, configContainer);
}
private void UpdateConfig(Configs.ConfigContainer configContainer, MD5Hash hash)
{
if (configContainer?.CDNConfig == null)
{
return;
}
// determine the field name
string identifier;
if (IsGroupIndex)
{
identifier = IsPatchIndex ? "patch-archive-group" : "archive-group";
}
else if (IsLooseIndex)
{
identifier = IsPatchIndex ? "patch-file-index" : "file-index";
}
else
{
identifier = IsPatchIndex ? "patch-archives" : "archives";
}
// update the collection
var collection = configContainer.CDNConfig.GetValues(identifier);
if (collection != null)
{
if (IsGroupIndex)
{
collection[0] = hash.ToString(); // group indicies are single entries
}
else
{
collection.Remove(Checksum.ToString()); // all others are collections
collection.Add(hash.ToString());
}
}
// TODO sizes - not sure how these are calculated
}
/// <summary>
/// Returns a list of index filenames used by the specific CDNConfig
/// </summary>
/// <param name="configContainer"></param>
/// <returns></returns>
private HashSet <string> GetRequiredIndices(Configs.ConfigContainer configContainer)
{
var indices = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// data archives
var archives = configContainer.CDNConfig.GetValues("archives");
if (archives != null)
{
indices.UnionWith(archives);
}
// patch archives
var patcharchives = configContainer.CDNConfig.GetValues("patch-archives");
if (patcharchives != null)
{
indices.UnionWith(patcharchives);
}
// loose file index
var fileIndex = configContainer.CDNConfig.GetValue("file-index");
if (fileIndex != null)
{
indices.Add(fileIndex);
}
// loose patch file index
var patchIndex = configContainer.CDNConfig.GetValue("patch-file-index");
if (patchIndex != null)
{
indices.Add(patchIndex);
}
return(indices);
}
/// <summary>
/// Saves the IndexFile to disk and optionally updates the CDN config
/// </summary>
/// <param name="directory"></param>
/// <param name="configContainer"></param>
public void Write(string directory, Configs.ConfigContainer configContainer = null)
{
RequiresSave = false;
// TODO patch index writing
if (IsPatchIndex || Type == IndexType.Unknown)
{
throw new NotImplementedException();
}
// Group Indicies only supported for Data and Patch indicies
if (IsGroupIndex && IsLooseIndex)
{
throw new NotImplementedException();
}
List <MD5Hash> EKeyLookupHashes = new List <MD5Hash>();
List <MD5Hash> PageChecksums = new List <MD5Hash>();
// update Footer
IndexFooter.EntryCount = (uint)_indexEntries.Count;
// get file dimensions
var(PageSize, EntriesPerPage, PageCount) = GetFileDimensions();
using var md5 = MD5.Create();
using var ms = new MemoryStream(PageCount * (PageSize + 1));
using var bw = new BinaryWriter(ms);
// set capcity
EKeyLookupHashes.Capacity = PageCount;
PageChecksums.Capacity = PageCount;
// IndexEntries
int index = 0;
for (int i = 0; i < PageCount; i++)
{
// write the entries
for (int j = 0; j < EntriesPerPage && index < IndexFooter.EntryCount; j++)
{
_indexEntries.Values[index++].Write(bw, IndexFooter);
}
// apply padding and store EKey and page checksum
int remainder = (int)bw.BaseStream.Position % PageSize;
if (remainder > 0)
{
ms.Write(new byte[PageSize - remainder]);
}
EKeyLookupHashes.Add(_indexEntries.Values[index - 1].Key);
PageChecksums.Add(ms.HashSlice(md5, bw.BaseStream.Position - PageSize, PageSize, IndexFooter.ChecksumSize));
}
// EKey Lookup
long lookupStartPos = bw.BaseStream.Position;
foreach (var lookupHash in EKeyLookupHashes)
{
bw.Write(lookupHash.Value);
}
// Page hashes - final page is ignored
long pageStartPos = bw.BaseStream.Position;
PageChecksums.RemoveAt(PageChecksums.Count - 1);
foreach (var pagechecksum in PageChecksums)
{
bw.Write(pagechecksum.Value);
}
// LastPage hash - last PageSize of Entries
long footerStartPos = bw.BaseStream.Position;
IndexFooter.LastPageHash = ms.HashSlice(md5, lookupStartPos - PageSize, PageSize, IndexFooter.ChecksumSize);
bw.Write(IndexFooter.LastPageHash.Value);
// TOC hash - from EKey Lookup to LastPage Hash
IndexFooter.ContentsHash = ms.HashSlice(md5, lookupStartPos, ms.Length - lookupStartPos, IndexFooter.ChecksumSize);
bw.Write(IndexFooter.ContentsHash.Value);
// write footer
IndexFooter.Write(bw);
// compute filename - from ContentsHash to EOF
MD5Hash newChecksum = ms.HashSlice(md5, footerStartPos + IndexFooter.ChecksumSize, IndexFooter.Size - IndexFooter.ChecksumSize);
// update the CDN Config
UpdateConfig(configContainer, newChecksum, bw.BaseStream.Length);
//// remove old index file
//if (!Checksum.IsEmpty)
// Helpers.Delete(Checksum.ToString() + ".index", directory);
// update Checksum
Checksum = newChecksum;
// Group Indicies are generated client-side
if (IsGroupIndex)
{
return;
}
string saveLocation = Helpers.GetCDNPath(Checksum.ToString() + ".index", "data", directory, true);
if (!File.Exists(saveLocation))
{
// save to disk
File.WriteAllBytes(saveLocation, ms.ToArray());
}
}
