/// <summary>
/// Updates modified data indices and writes enqueued files to archives
/// <para>Note: IndexFile saving is limited to new entries if the container was opened remotely</para>
/// </summary>
/// <param name="directory"></param>
/// <param name="dispose">Delete old files</param>
/// <param name="configContainer"></param>
public void Save(string directory, Configs.ConfigContainer configContainer = null)
{
bool sameDirectory = directory.EqualsIC(_sourceDirectory);
// save altered Data archive indices
if (!IsRemote)
{
foreach (var index in DataIndices)
{
if (index.IsGroupIndex)
{
continue;
}
if (index.RequiresSave)
{
// save the index file and blob
string prevBlob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
index.Write(directory, configContainer);
index.WriteBlob(directory, prevBlob);
}
else if (!sameDirectory)
{
// copy the index file and blob
string oldblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
string newblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", directory, true);
File.Copy(oldblob, newblob);
File.Copy(oldblob + ".index", newblob + ".index");
}
}
}
// prevent duplicated entries
var duplicates = QueuedEntries.Keys
.Where(k => GetIndexFileAndEntry(IndexType.Data, k, out _) != null)
.ToArray();
foreach (var key in duplicates)
{
QueuedEntries.Remove(key);
}
// create any new archive indices
var partitions = EnumerablePartitioner.ConcreteBatch(QueuedEntries.Values, ArchiveDataSize, (x) => x.EBlock.CompressedSize);
foreach (var entries in partitions)
{
IndexFile index = new IndexFile(IndexType.Data);
index.Add(entries);
index.Write(directory, configContainer);
index.WriteBlob(directory);
}
// reload indices
Open(directory, useParallelism: _useParallelism);
}
/// <summary>
/// Updates modified data indices and writes enqueued files to archives
/// </summary>
/// <param name="directory"></param>
/// <param name="dispose">Delete old files</param>
/// <param name="configContainer"></param>
public void Save(string directory, Configs.ConfigContainer configContainer = null)
{
bool sameDirectory = directory.Equals(_sourceDirectory, StringComparison.OrdinalIgnoreCase);
// save altered Data archive indices
foreach (var index in DataIndices)
{
if (index.IsGroupIndex)
{
continue;
}
if (index.RequiresSave)
{
// save the index file and blob
string prevBlob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
index.Write(directory, configContainer);
index.WriteBlob(directory, prevBlob);
}
else if (!sameDirectory)
{
// copy the index file and blob
string oldblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", _sourceDirectory);
string newblob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", directory, true);
File.Copy(oldblob, newblob);
File.Copy(oldblob + ".index", newblob + ".index");
}
}
// create any new archive indices
var partitions = EnumerablePartitioner.ConcreteBatch(_fileQueue.Values, ArchiveDataSize, (x) => x.EBlock.CompressedSize);
foreach (var entries in partitions)
{
IndexFile index = new IndexFile(IndexType.Data);
index.Add(entries);
index.Write(directory, configContainer);
index.WriteBlob(directory);
}
// TODO 1. verify if this is required 2. fix
// compute the Data Index Group hash
//GenerateIndexGroup(directory, configContainer);
// reload indices
_indices.Clear();
Open(directory, _useParallelism);
}
/// <summary>
/// Writes the page data and calculates the page lookups
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="bw"></param>
/// <param name="pageSize"></param>
/// <param name="container"></param>
/// <returns></returns>
private PageIndexTable WritePageImpl <T>(BinaryWriter bw, int pageSize, IDictionary <MD5Hash, T> container) where T : EncodingEntryBase
{
PageIndexTable pageIndices = new PageIndexTable();
bool EOFflag = typeof(T) == typeof(EncodingEncodedEntry);
using (var md5 = MD5.Create())
{
// split entries into pages of pageSize
var pages = EnumerablePartitioner.ConcreteBatch(container.Values, pageSize, (x) => x.Size);
uint pageCount = (uint)pages.Count();
// set Header PageCount
EncodingHeader.SetPageCount <T>(pageCount);
uint index = pageCount;
foreach (var page in pages)
{
// write page entries and pad to pageSize
page.ForEach(x => x.Write(bw, EncodingHeader));
// apply EOF flag (EKey page)
if (EOFflag && --index == 0)
{
bw.Write(new byte[EncodingHeader.EKeyHashSize]);
bw.Write(0xFFFFFFFF);
}
// pad to page size
bw.Write(new byte[pageSize - (bw.BaseStream.Position % pageSize)]);
// create page index record
pageIndices[page[0].Key] = bw.BaseStream.HashSlice(md5, bw.BaseStream.Position - pageSize, pageSize);
page.Clear();
}
}
return(pageIndices);
}