void GenerateCsvTableText <TClass>(
CsvHelper.CsvReader csv,
io.Compression.ZipArchiveEntry entry,
DbTable table,
string rootPath)
{
string pagerName = $"{table.Name}.pager";
string pagerFilename = io.Path.Combine(rootPath, $"{pagerName}.txt");
var pageCount = 1;
var pageSize = table.Pager.PagerSize;
//set to 0 initially, but now starts with "1"
var globalRow = 1;
var position = 0;
var prevPosition = 0;
var prevGlobalRow = 0;
//page size calculator
Func <int, int, int> calculatePageSize = (prePos, currPos) => currPos - prePos; // + 1;
using (var csvPagerwriter = new io.StreamWriter(io.File.Create(pagerFilename)))
{
IndexStruct[] indexesArray = null;
//CSV text writer
io.StreamWriter csvwriter = null;
if (!table.Generate)
{
Console.WriteLine($"");
}
else
{
//CSV text writer
csvwriter = new io.StreamWriter(io.File.Create(io.Path.Combine(rootPath, $"bin\\{table.Name}.{CsvDb.SchemaTableDefaultExtension}")));
Console.WriteLine($" creating temporary text index files.");
//create temporary files for csv pager, and all indexes
indexesArray = (from col in table.Columns
where col.Indexed
let tmpfile = io.Path.Combine(rootPath, $"{col.Indexer}.txt")
select new IndexStruct
{
column = col,
//csv
file = tmpfile,
writer = new io.StreamWriter(io.File.Create(tmpfile))
}).ToArray();
Console.WriteLine($" ({indexesArray.Length}) index(es) found.");
Console.WriteLine($"{String.Join("", indexesArray.Select(i => $" -{i.file}\r\n"))}");
}
if (!csv.Read())
{
Console.WriteLine($" error: table header expected!");
}
Console.WriteLine($" CSV headers read.");
//check field headers indexes with table metadata
var ndx = 0;
foreach (var col in table.Columns)
{
var headcol = csv.GetField(ndx);
if (headcol != col.Name || ndx != col.Index)
{
Console.WriteLine($" error: table header name and/or ordering is invalid!");
return;
}
ndx++;
}
Console.WriteLine($" CSV headers checked.\r\n Writing CSV database text file, and generating index files...");
//
var tableUniqueKeyCol = table.Columns.Where(c => c.Key).First();
//
int rowLine = 0;
var sb = new StringBuilder();
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
var csvParser = new DbRecordParser(csv, table);
//read each record of csv entry
while (csv.Read())
{
rowLine++;
//
if (table.Generate)
{
//get csv record columns string
csvParser.ReadRecord();
var csvRow = csvParser.Record();
//write line to temporary CSV buffer
sb.Append($"{csvRow}\r\n");
if (sb.Length > 16 * 1024)
{
//write line to the .CSV file
csvwriter.Write(sb.ToString());
sb.Length = 0;
}
//get unique key value (non-casted)
var tableUniqueKeyValue = csvParser.Values[tableUniqueKeyCol.Index];
//process pager
if (pageSize-- <= 0)
{
pageCount++;
//reset page size
pageSize = table.Pager.PagerSize;
//save previous one
//page header
//[start row]
//[start position]
csvPagerwriter.WriteLine(
$"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, position)}");
//
prevGlobalRow = globalRow;
prevPosition = position;
}
//process indexes
foreach (var index in indexesArray)
{
var indexColValue = csvParser.Values[index.column.Index];
//position index: $"{value}|{position}"
//row index: $"{value}|{globalRow}"
//if index is key, store its position
// otherwise its line number 1-based must be the key value for single key table.
string indexCsvLine = null;
if (index.column.Key)
{
//for Keys store the position inside the .CSV file
indexCsvLine = $"{indexColValue}|{position}";
}
else
{
//do it too here, so no need to read key tree
//indexLine = $"{indexColValue}|{tableUniqueKeyValue}";
indexCsvLine = $"{indexColValue}|{position}";
}
//write line
index.writer.WriteLine(indexCsvLine);
}
//calculate values for next row
globalRow++;
position += csvRow.Length + 2; // +2 for \r\n
}
}
;
//write missing rows if any
if (sb.Length > 0)
{
//write line to the .CSV file
csvwriter.Write(sb.ToString());
sb.Length = 0;
}
//ellapsed time
sw.Stop();
Console.WriteLine("ellapsed {0} ms", sw.ElapsedMilliseconds);
//var timespan = DateTime.Now - startTime;
//var ellapsed = $" ellapsed: {timespan}";
//Console.WriteLine(ellapsed);
//store/update line count
table.Rows = rowLine;
table.Pager.Count = pageCount;
table.Pager.File = pagerName;
//save last page
if (table.Generate)
{
csvPagerwriter.WriteLine(
$"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, position)}");
}
Console.WriteLine($" ({rowLine}) line(s) processed.");
//close all index writers, and delete temporary files
indexesArray?.ToList().ForEach(index =>
{
index.writer.Dispose();
});
csvwriter?.Dispose();
}
if (!table.Generate)
{
Console.WriteLine($" indexers were not generated!");
}
}
void GenerateBinTableText <TClass>(
CsvHelper.CsvReader csv,
io.Compression.ZipArchiveEntry entry,
DbTable table,
string rootPath)
{
string pagerName = $"{table.Name}.pager";
string pagerFilename = io.Path.Combine(rootPath, $"{pagerName}.txt");
var pageCount = 1;
var pageSize = table.Pager.PagerSize;
//set to 0 initially, but now starts with "1"
var globalRow = 1;
var prevPosition = 0;
var prevGlobalRow = 0;
var binaryPosition = 0;
var stream = new io.MemoryStream();
var bufferWriter = new io.BinaryWriter(stream);
var columnTypes = table.ColumnTypes;
//page size calculator
Func <int, int, int> calculatePageSize = (prePos, currPos) => currPos - prePos; // + 1;
using (var csvPagerwriter = new io.StreamWriter(io.File.Create(pagerFilename)))
{
IndexStruct[] indexesArray = null;
//Bin writer
io.BinaryWriter binWriter = null;
if (table.Generate)
{
//Bin writer
binWriter = new io.BinaryWriter(io.File.Create(io.Path.Combine(rootPath, $"bin\\{table.Name}.{CsvDb.SchemaTableDataExtension}")));
//save column mask
table.RowMaskLength = Math.DivRem(table.Count, 8, out int remainder);
int bits = table.RowMaskLength * 8 + remainder;
table.RowMask = (UInt64)Math.Pow(2, bits - 1);
if (remainder != 0)
{
table.RowMaskLength++;
}
Console.WriteLine($" creating temporary text index files.");
//create temporary files for csv pager, and all indexes
indexesArray = (from col in table.Columns
where col.Indexed
let tmpfile = io.Path.Combine(rootPath, $"{col.Indexer}.bin.txt")
select new IndexStruct
{
column = col,
//csv
file = tmpfile,
writer = new io.StreamWriter(io.File.Create(tmpfile))
}).ToArray();
Console.WriteLine($" ({indexesArray.Length}) index(es) found.");
Console.WriteLine($"{String.Join("", indexesArray.Select(i => $" -{i.file}\r\n"))}");
}
if (!csv.Read())
{
Console.WriteLine($" error: table header expected!");
}
Console.WriteLine($" CSV headers read.");
//check field headers indexes with table metadata
var ndx = 0;
foreach (var col in table.Columns)
{
var headcol = csv.GetField(ndx);
if (headcol != col.Name || ndx != col.Index)
{
Console.WriteLine($" error: table header name and/or ordering is invalid!");
return;
}
ndx++;
}
Console.WriteLine($" CSV headers checked.\r\n Writing Bin database text file, and generating index files...");
//
var tableUniqueKeyCol = table.Columns.Where(c => c.Key).First();
//
int rowLine = 0;
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
var csvParser = new DbRecordParser(csv, table);
//read each record of csv entry
while (csv.Read())
{
rowLine++;
//
if (table.Generate)
{
//get csv record columns string
csvParser.ReadRecord();
//get unique key value (non-casted)
var tableUniqueKeyValue = csvParser.Values[tableUniqueKeyCol.Index];
//process pager
if (pageSize-- <= 0)
{
pageCount++;
//reset page size
pageSize = table.Pager.PagerSize;
//save previous one
//page header
//[start row]
//[start position]
csvPagerwriter.WriteLine(
$"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, binaryPosition)}");
//
prevGlobalRow = globalRow;
prevPosition = binaryPosition;
}
//process indexes
foreach (var index in indexesArray)
{
var indexColValue = csvParser.Values[index.column.Index];
//position index: $"{value}|{position}"
//row index: $"{value}|{globalRow}"
//if index is key, store its position
// otherwise its line number 1-based must be the key value for single key table.
string indexBinLine = null;
if (index.column.Key)
{
//for Keys store the position inside the .CSV file
indexBinLine = $"{indexColValue}|{binaryPosition}";
}
else
{
//do it too here, so no need to read key tree
//indexLine = $"{indexColValue}|{tableUniqueKeyValue}";
indexBinLine = $"{indexColValue}|{binaryPosition}";
}
//write line
index.writer.WriteLine(indexBinLine);
}
//save binary data
UInt64 flags = 0;
var columnBit = table.RowMask;
stream.Position = 0;
foreach (var col in table.Columns)
{
var _ndx = col.Index;
var textValue = csvParser.Values[_ndx];
//parser get Empty string when should be null
if (String.IsNullOrEmpty(textValue))
{
textValue = null;
}
var colType = columnTypes[_ndx];
if (textValue == null)
{
//signal only the null flag as true
flags |= columnBit;
}
else
{
var throwException = false;
switch (colType)
{
case DbColumnType.Char:
char charValue = (char)0;
throwException = !Char.TryParse(textValue, out charValue);
//write
bufferWriter.Write(charValue);
break;
case DbColumnType.Byte:
byte byteValue = 0;
throwException = !Byte.TryParse(textValue, out byteValue);
//write
bufferWriter.Write(byteValue);
break;
case DbColumnType.Int16:
Int16 int16Value = 0;
throwException = !Int16.TryParse(textValue, out int16Value);
//write
bufferWriter.Write(int16Value);
break;
case DbColumnType.Int32:
Int32 int32Value = 0;
throwException = !Int32.TryParse(textValue, out int32Value);
//write
bufferWriter.Write(int32Value);
break;
case DbColumnType.Single:
float floatValue = 0.0f;
throwException = !float.TryParse(textValue, out floatValue);
//write
bufferWriter.Write(floatValue);
break;
case DbColumnType.Double:
Double doubleValue = 0.0;
throwException = !Double.TryParse(textValue, out doubleValue);
//write
bufferWriter.Write(doubleValue);
break;
case DbColumnType.Decimal:
Decimal decimalValue = 0;
throwException = !Decimal.TryParse(textValue, out decimalValue);
//write
bufferWriter.Write(decimalValue);
break;
case DbColumnType.String:
//write
bufferWriter.Write(textValue);
break;
default:
throw new ArgumentException($"unsupported type on {col.Name}.{colType} row: {rowLine}");
}
if (throwException)
{
throw new ArgumentException($"unable to cast: {textValue} on {col.Name}.{colType} row: {rowLine}");
}
}
//shift right column Bit until it reaches 0 -the last column rightmost
columnBit >>= 1;
}
//write true binary record
var flagsBuffer = BitConverter.GetBytes(flags);
binWriter.Write(flagsBuffer, 0, table.RowMaskLength);
//write non-null records
var recBinary = stream.ToArray();
binWriter.Write(recBinary, 0, recBinary.Length);
//update binary position for next row
binaryPosition += table.RowMaskLength + recBinary.Length;
}
}
;
//ellapsed time
sw.Stop();
Console.WriteLine("ellapsed {0} ms", sw.ElapsedMilliseconds);
//var timespan = DateTime.Now - startTime;
//var ellapsed = $" ellapsed: {timespan}";
//Console.WriteLine(ellapsed);
//store/update line count
table.Rows = rowLine;
table.Pager.Count = pageCount;
table.Pager.File = pagerName;
//save last page
if (table.Generate)
{
csvPagerwriter.WriteLine(
$"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, binaryPosition)}");
}
Console.WriteLine($" ({rowLine}) line(s) processed.");
//close all index writers, and delete temporary files
indexesArray?.ToList().ForEach(index =>
{
index.writer.Dispose();
});
binWriter?.Dispose();
}
if (!table.Generate)
{
Console.WriteLine($" indexers were not generated!");
}
}