public ArchiveTable(JObject json, ParquetReader reader, ArchiveTableInformation tableInformation, string name) : base(json) { Name = name; _reader = new RememberingParquetReader(reader); IsSaved = false; AddColumns(tableInformation); }
public async Task <bool> WriteData(JObject root, ISessionWriter writer) { var pathArr = Meta["attachments"].ToObject <string[]>() ?? throw new ArgumentException("Table is missing 'attachments'"); //TODO: Implement? if (false && IsSaved) { var stream = await _archive.OpenFile(_zipEntry); writer.StoreFileId(stream, pathArr[0]); } else { // This stream will be disposed by the sessionWriter var ms = new MemoryStream(); // Make a copy of the Remembering reader that later can be discarded // This to avoid to read in all tables in memory at the same time. var fullReader = new RememberingParquetReader(_reader); fullReader.LoadAll(); using (var tableWriter = new ParquetWriter(fullReader.Schema, ms)) { using (var rowGroup = tableWriter.CreateRowGroup()) // Using construction assure correct storage of final rowGroup details in parquet file { foreach (var field in fullReader.Schema.GetDataFields()) { var column = new DataColumn(field, fullReader.GetColumn(field)); rowGroup.WriteColumn(column); } } } ms.Position = 0; writer.StoreFileId(ms, pathArr[0]); } // TODO AUTOACTIVE-58 - Generalize copy of previous metadata for save // Copy previous root["meta"] = Meta; root["user"] = User; // Overwrite potentially changed // TODO root["meta"]["is_world_clock"] = ; // TODO root["meta"]["synced_to"] = ; return(true); }
internal ArchiveTable(JObject json, Archive.Archive archive, Guid sessionId, ArchiveTableInformation tableInformation) : base(json) { IsSaved = true; _archive = archive; _zipEntry = tableInformation.ZipEntry; _sessionId = sessionId; if (tableInformation.Time == null) { throw new ArgumentException("Table does not have a column named 'Time'"); } var streamTask = archive.OpenFile(_zipEntry); streamTask.Wait(); using (var reader = new ParquetReader(streamTask.Result)) { _reader = new RememberingParquetReader(reader); } AddColumns(tableInformation); }
private static Task <T[]> GenerateLoader <T>(RememberingParquetReader reader, DataField column) { return(new Task <T[]>(() => LoadColumn <T>(reader, column).Result)); }
private static Task <T[]> LoadColumn <T>(RememberingParquetReader reader, DataField column) { return(Task.FromResult(reader.LoadColumn <T>(column))); }
public RememberingParquetReader(RememberingParquetReader rpr) { // Make a copy of existing data and reader _reader = rpr._reader; _data = new Dictionary <DataField, Array>(rpr._data); }