/// <summary>
/// Loads a line of TSV data into this object.
/// Ignores fields which have no mappings currently.
/// Length match between header and line has to be guaranteed, because
/// this method doesn't know enough for a nice error message.
/// </summary>
/// <param name="TsvMapping"></param>
/// <param name="record">Target object</param>
/// <param name="line">Data from a TSV line per field</param>
internal void LoadFromTSVLine(TsvMapping <T> TsvMapping, object record, List <string> line)
{
for (int propertyIndex = 0; propertyIndex < Properties.Length; propertyIndex++)
{
Nullable <int> lineIndex = TsvMapping.FromRecordToTsv[propertyIndex];
if (lineIndex == null)
{
Properties[propertyIndex].SetValue(record, null);
}
else
{
Properties[propertyIndex].SetValue(record, ConvertStringToValue(propertyIndex, line[(int)lineIndex]));
}
}
}
/// <summary>
/// Decompress the raw data and desirialize the records.
/// </summary>
/// <param name="queryPlan">Optional for queries. Filters the recrods if passed.</param>
internal void DeserializeDecompress(QueryPlan <T> queryPlan = null)
{
if (rawCompressedData == null)
{
return;
}
List <string> header = null;
TsvMapping <T> tsvMapping = null;
using var ms = new MemoryStream(rawCompressedData);
using var gzip = new GZipStream(ms, CompressionMode.Decompress);
using var tsv = new TsvReader(gzip);
int lineCount = 0;
Nullable <int> tsvColumnIndex = null;
bool isFirstLine = true;
bool notMatching = false;
while (!tsv.EndOfStream)
{
var line = tsv.ReadLine();
if (isFirstLine)
{
// Use its own header for compatibility.
header = line;
isFirstLine = false;
tsvMapping = new TsvMapping <T>(table, header);
continue;
}
if (header.Count != line.Count)
{
throw new Exception($"Header length and column count mismatch in file '{FullPath}' at line {lineCount + 1}.");
}
/*
* Filtering by 'Where' expressions
*/
if (queryPlan != null)
{
notMatching = false;
foreach (var filter in queryPlan.FreeIndexFilters)
{
tsvColumnIndex = tsvMapping.FromRecordToTsv[filter.Key];
if (tsvColumnIndex == null || line[(int)tsvColumnIndex] != filter.Value)
{
notMatching = true;
break;
}
}
if (notMatching)
{
continue;
}
}
var record = new T();
table.LoadFromTSVLine(tsvMapping, record, line);
/*
* Filtering by flex expressions
*/
if (queryPlan != null)
{
notMatching = false;
foreach (var exp in queryPlan.Query.FlexFilters)
{
if (!exp(record))
{
notMatching = true;
break;
}
}
if (notMatching)
{
continue;
}
}
string unique = table.GetUnique(record);
this.data[unique] = record;
this.lines.Add(record);
lineCount++;
}
/*
* Sort records inside the packet
*/
if (queryPlan != null)
{
if (queryPlan.FreeSorting.Count > 0)
{
var props = table.Properties.ToArray();
this.lines.Sort((x, y) => {
foreach (var directive in queryPlan.FreeSorting)
{
int dir = directive.Item2 == SortingDirection.Ascending ? 1 : -1;
var valueX = props[directive.Item1].GetValue(x);
var valueY = props[directive.Item1].GetValue(y);
if (valueX == null && valueY == null)
{
continue;
}
if (valueX == null)
{
return(-dir);
}
if (valueY == null)
{
return(dir);
}
return(((IComparable)valueX).CompareTo((IComparable)valueY) * dir);
}
return(0);
});
}
}
}