public ArchiveTable(JObject json, ParquetReader reader, ArchiveTableInformation tableInformation, string name) :
base(json)
{
Name = name;
_reader = new RememberingParquetReader(reader);
IsSaved = false;
AddColumns(tableInformation);
}
public async Task <bool> WriteData(JObject root, ISessionWriter writer)
{
var pathArr = Meta["attachments"].ToObject <string[]>() ?? throw new ArgumentException("Table is missing 'attachments'");
//TODO: Implement?
if (false && IsSaved)
{
var stream = await _archive.OpenFile(_zipEntry);
writer.StoreFileId(stream, pathArr[0]);
}
else
{
// This stream will be disposed by the sessionWriter
var ms = new MemoryStream();
// Make a copy of the Remembering reader that later can be discarded
// This to avoid to read in all tables in memory at the same time.
var fullReader = new RememberingParquetReader(_reader);
fullReader.LoadAll();
using (var tableWriter = new ParquetWriter(fullReader.Schema, ms))
{
using (var rowGroup = tableWriter.CreateRowGroup()) // Using construction assure correct storage of final rowGroup details in parquet file
{
foreach (var field in fullReader.Schema.GetDataFields())
{
var column = new DataColumn(field, fullReader.GetColumn(field));
rowGroup.WriteColumn(column);
}
}
}
ms.Position = 0;
writer.StoreFileId(ms, pathArr[0]);
}
// TODO AUTOACTIVE-58 - Generalize copy of previous metadata for save
// Copy previous
root["meta"] = Meta;
root["user"] = User;
// Overwrite potentially changed
// TODO root["meta"]["is_world_clock"] = ;
// TODO root["meta"]["synced_to"] = ;
return(true);
}
internal ArchiveTable(JObject json, Archive.Archive archive, Guid sessionId, ArchiveTableInformation tableInformation) :
base(json)
{
IsSaved = true;
_archive = archive;
_zipEntry = tableInformation.ZipEntry;
_sessionId = sessionId;
if (tableInformation.Time == null)
{
throw new ArgumentException("Table does not have a column named 'Time'");
}
var streamTask = archive.OpenFile(_zipEntry);
streamTask.Wait();
using (var reader = new ParquetReader(streamTask.Result))
{
_reader = new RememberingParquetReader(reader);
}
AddColumns(tableInformation);
}
private static Task <T[]> GenerateLoader <T>(RememberingParquetReader reader, DataField column)
{
return(new Task <T[]>(() => LoadColumn <T>(reader, column).Result));
}
private static Task <T[]> LoadColumn <T>(RememberingParquetReader reader, DataField column)
{
return(Task.FromResult(reader.LoadColumn <T>(column)));
}
public RememberingParquetReader(RememberingParquetReader rpr)
{
// Make a copy of existing data and reader
_reader = rpr._reader;
_data = new Dictionary <DataField, Array>(rpr._data);
}
