Beispiel #1
0
        void GenerateCsvTableText <TClass>(
            CsvHelper.CsvReader csv,
            io.Compression.ZipArchiveEntry entry,
            DbTable table,
            string rootPath)
        {
            string pagerName     = $"{table.Name}.pager";
            string pagerFilename = io.Path.Combine(rootPath, $"{pagerName}.txt");
            var    pageCount     = 1;
            var    pageSize      = table.Pager.PagerSize;
            //set to 0 initially, but now starts with "1"
            var globalRow = 1;
            var position  = 0;

            var prevPosition  = 0;
            var prevGlobalRow = 0;

            //page size calculator
            Func <int, int, int> calculatePageSize = (prePos, currPos) => currPos - prePos;            // + 1;

            using (var csvPagerwriter = new io.StreamWriter(io.File.Create(pagerFilename)))
            {
                IndexStruct[] indexesArray = null;

                //CSV text writer
                io.StreamWriter csvwriter = null;

                if (!table.Generate)
                {
                    Console.WriteLine($"");
                }
                else
                {
                    //CSV text writer
                    csvwriter = new io.StreamWriter(io.File.Create(io.Path.Combine(rootPath, $"bin\\{table.Name}.{CsvDb.SchemaTableDefaultExtension}")));

                    Console.WriteLine($"  creating temporary text index files.");
                    //create temporary files for csv pager, and all indexes
                    indexesArray = (from col in table.Columns
                                    where col.Indexed
                                    let tmpfile = io.Path.Combine(rootPath, $"{col.Indexer}.txt")
                                                  select new IndexStruct
                    {
                        column = col,
                        //csv
                        file = tmpfile,
                        writer = new io.StreamWriter(io.File.Create(tmpfile))
                    }).ToArray();
                    Console.WriteLine($"   ({indexesArray.Length}) index(es) found.");
                    Console.WriteLine($"{String.Join("", indexesArray.Select(i => $"    -{i.file}\r\n"))}");
                }
                if (!csv.Read())
                {
                    Console.WriteLine($"  error: table header expected!");
                }
                Console.WriteLine($"  CSV headers read.");

                //check field headers indexes with table metadata
                var ndx = 0;
                foreach (var col in table.Columns)
                {
                    var headcol = csv.GetField(ndx);
                    if (headcol != col.Name || ndx != col.Index)
                    {
                        Console.WriteLine($"  error: table header name and/or ordering is invalid!");
                        return;
                    }
                    ndx++;
                }
                Console.WriteLine($"  CSV headers checked.\r\n  Writing CSV database text file, and generating index files...");
                //
                var tableUniqueKeyCol = table.Columns.Where(c => c.Key).First();
                //
                int rowLine = 0;
                var sb      = new StringBuilder();
                var sw      = new System.Diagnostics.Stopwatch();
                sw.Start();
                var csvParser = new DbRecordParser(csv, table);

                //read each record of csv entry
                while (csv.Read())
                {
                    rowLine++;
                    //
                    if (table.Generate)
                    {
                        //get csv record columns string
                        csvParser.ReadRecord();
                        var csvRow = csvParser.Record();

                        //write line to temporary CSV buffer
                        sb.Append($"{csvRow}\r\n");

                        if (sb.Length > 16 * 1024)
                        {
                            //write line to the .CSV file
                            csvwriter.Write(sb.ToString());
                            sb.Length = 0;
                        }
                        //get unique key value (non-casted)
                        var tableUniqueKeyValue = csvParser.Values[tableUniqueKeyCol.Index];

                        //process pager
                        if (pageSize-- <= 0)
                        {
                            pageCount++;
                            //reset page size
                            pageSize = table.Pager.PagerSize;

                            //save previous one
                            //page header
                            //[start row]
                            //[start position]
                            csvPagerwriter.WriteLine(
                                $"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, position)}");
                            //
                            prevGlobalRow = globalRow;
                            prevPosition  = position;
                        }

                        //process indexes
                        foreach (var index in indexesArray)
                        {
                            var indexColValue = csvParser.Values[index.column.Index];

                            //position index: $"{value}|{position}"
                            //row index:			$"{value}|{globalRow}"
                            //if index is key, store its position
                            //  otherwise its line number 1-based   must be the key value for single key table.
                            string indexCsvLine = null;

                            if (index.column.Key)
                            {
                                //for Keys store the position inside the .CSV file
                                indexCsvLine = $"{indexColValue}|{position}";
                            }
                            else
                            {
                                //do it too here, so no need to read key tree
                                //indexLine = $"{indexColValue}|{tableUniqueKeyValue}";
                                indexCsvLine = $"{indexColValue}|{position}";
                            }
                            //write line
                            index.writer.WriteLine(indexCsvLine);
                        }
                        //calculate values for next row
                        globalRow++;
                        position += csvRow.Length + 2;                         // +2 for \r\n
                    }
                }
                ;
                //write missing rows if any
                if (sb.Length > 0)
                {
                    //write line to the .CSV file
                    csvwriter.Write(sb.ToString());
                    sb.Length = 0;
                }

                //ellapsed time
                sw.Stop();
                Console.WriteLine("ellapsed {0} ms", sw.ElapsedMilliseconds);
                //var timespan = DateTime.Now - startTime;
                //var ellapsed = $"  ellapsed: {timespan}";
                //Console.WriteLine(ellapsed);

                //store/update line count
                table.Rows        = rowLine;
                table.Pager.Count = pageCount;
                table.Pager.File  = pagerName;

                //save last page
                if (table.Generate)
                {
                    csvPagerwriter.WriteLine(
                        $"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, position)}");
                }

                Console.WriteLine($"  ({rowLine}) line(s) processed.");

                //close all index writers, and delete temporary files
                indexesArray?.ToList().ForEach(index =>
                {
                    index.writer.Dispose();
                });
                csvwriter?.Dispose();
            }
            if (!table.Generate)
            {
                Console.WriteLine($"  indexers were not generated!");
            }
        }
Beispiel #2
0
        void GenerateBinTableText <TClass>(
            CsvHelper.CsvReader csv,
            io.Compression.ZipArchiveEntry entry,
            DbTable table,
            string rootPath)
        {
            string pagerName     = $"{table.Name}.pager";
            string pagerFilename = io.Path.Combine(rootPath, $"{pagerName}.txt");
            var    pageCount     = 1;
            var    pageSize      = table.Pager.PagerSize;
            //set to 0 initially, but now starts with "1"
            var globalRow = 1;

            var prevPosition  = 0;
            var prevGlobalRow = 0;

            var binaryPosition = 0;
            var stream         = new io.MemoryStream();
            var bufferWriter   = new io.BinaryWriter(stream);
            var columnTypes    = table.ColumnTypes;

            //page size calculator
            Func <int, int, int> calculatePageSize = (prePos, currPos) => currPos - prePos;            // + 1;

            using (var csvPagerwriter = new io.StreamWriter(io.File.Create(pagerFilename)))
            {
                IndexStruct[] indexesArray = null;
                //Bin writer
                io.BinaryWriter binWriter = null;

                if (table.Generate)
                {
                    //Bin writer
                    binWriter = new io.BinaryWriter(io.File.Create(io.Path.Combine(rootPath, $"bin\\{table.Name}.{CsvDb.SchemaTableDataExtension}")));

                    //save column mask
                    table.RowMaskLength = Math.DivRem(table.Count, 8, out int remainder);
                    int bits = table.RowMaskLength * 8 + remainder;
                    table.RowMask = (UInt64)Math.Pow(2, bits - 1);
                    if (remainder != 0)
                    {
                        table.RowMaskLength++;
                    }

                    Console.WriteLine($"  creating temporary text index files.");
                    //create temporary files for csv pager, and all indexes
                    indexesArray = (from col in table.Columns
                                    where col.Indexed
                                    let tmpfile = io.Path.Combine(rootPath, $"{col.Indexer}.bin.txt")
                                                  select new IndexStruct
                    {
                        column = col,
                        //csv
                        file = tmpfile,
                        writer = new io.StreamWriter(io.File.Create(tmpfile))
                    }).ToArray();
                    Console.WriteLine($"   ({indexesArray.Length}) index(es) found.");
                    Console.WriteLine($"{String.Join("", indexesArray.Select(i => $"    -{i.file}\r\n"))}");
                }
                if (!csv.Read())
                {
                    Console.WriteLine($"  error: table header expected!");
                }
                Console.WriteLine($"  CSV headers read.");

                //check field headers indexes with table metadata
                var ndx = 0;
                foreach (var col in table.Columns)
                {
                    var headcol = csv.GetField(ndx);
                    if (headcol != col.Name || ndx != col.Index)
                    {
                        Console.WriteLine($"  error: table header name and/or ordering is invalid!");
                        return;
                    }
                    ndx++;
                }
                Console.WriteLine($"  CSV headers checked.\r\n  Writing Bin database text file, and generating index files...");
                //
                var tableUniqueKeyCol = table.Columns.Where(c => c.Key).First();
                //
                int rowLine = 0;
                var sw      = new System.Diagnostics.Stopwatch();
                sw.Start();
                var csvParser = new DbRecordParser(csv, table);

                //read each record of csv entry
                while (csv.Read())
                {
                    rowLine++;
                    //
                    if (table.Generate)
                    {
                        //get csv record columns string
                        csvParser.ReadRecord();

                        //get unique key value (non-casted)
                        var tableUniqueKeyValue = csvParser.Values[tableUniqueKeyCol.Index];

                        //process pager
                        if (pageSize-- <= 0)
                        {
                            pageCount++;
                            //reset page size
                            pageSize = table.Pager.PagerSize;

                            //save previous one
                            //page header
                            //[start row]
                            //[start position]
                            csvPagerwriter.WriteLine(
                                $"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, binaryPosition)}");
                            //
                            prevGlobalRow = globalRow;
                            prevPosition  = binaryPosition;
                        }

                        //process indexes
                        foreach (var index in indexesArray)
                        {
                            var indexColValue = csvParser.Values[index.column.Index];

                            //position index: $"{value}|{position}"
                            //row index:			$"{value}|{globalRow}"
                            //if index is key, store its position
                            //  otherwise its line number 1-based   must be the key value for single key table.
                            string indexBinLine = null;

                            if (index.column.Key)
                            {
                                //for Keys store the position inside the .CSV file
                                indexBinLine = $"{indexColValue}|{binaryPosition}";
                            }
                            else
                            {
                                //do it too here, so no need to read key tree
                                //indexLine = $"{indexColValue}|{tableUniqueKeyValue}";
                                indexBinLine = $"{indexColValue}|{binaryPosition}";
                            }
                            //write line
                            index.writer.WriteLine(indexBinLine);
                        }

                        //save binary data
                        UInt64 flags     = 0;
                        var    columnBit = table.RowMask;
                        stream.Position = 0;

                        foreach (var col in table.Columns)
                        {
                            var _ndx      = col.Index;
                            var textValue = csvParser.Values[_ndx];
                            //parser get Empty string when should be null
                            if (String.IsNullOrEmpty(textValue))
                            {
                                textValue = null;
                            }
                            var colType = columnTypes[_ndx];

                            if (textValue == null)
                            {
                                //signal only the null flag as true
                                flags |= columnBit;
                            }
                            else
                            {
                                var throwException = false;
                                switch (colType)
                                {
                                case DbColumnType.Char:
                                    char charValue = (char)0;
                                    throwException = !Char.TryParse(textValue, out charValue);
                                    //write
                                    bufferWriter.Write(charValue);
                                    break;

                                case DbColumnType.Byte:
                                    byte byteValue = 0;
                                    throwException = !Byte.TryParse(textValue, out byteValue);
                                    //write
                                    bufferWriter.Write(byteValue);
                                    break;

                                case DbColumnType.Int16:
                                    Int16 int16Value = 0;
                                    throwException = !Int16.TryParse(textValue, out int16Value);
                                    //write
                                    bufferWriter.Write(int16Value);
                                    break;

                                case DbColumnType.Int32:
                                    Int32 int32Value = 0;
                                    throwException = !Int32.TryParse(textValue, out int32Value);
                                    //write
                                    bufferWriter.Write(int32Value);
                                    break;

                                case DbColumnType.Single:
                                    float floatValue = 0.0f;
                                    throwException = !float.TryParse(textValue, out floatValue);
                                    //write
                                    bufferWriter.Write(floatValue);
                                    break;

                                case DbColumnType.Double:
                                    Double doubleValue = 0.0;
                                    throwException = !Double.TryParse(textValue, out doubleValue);
                                    //write
                                    bufferWriter.Write(doubleValue);
                                    break;

                                case DbColumnType.Decimal:
                                    Decimal decimalValue = 0;
                                    throwException = !Decimal.TryParse(textValue, out decimalValue);
                                    //write
                                    bufferWriter.Write(decimalValue);
                                    break;

                                case DbColumnType.String:
                                    //write
                                    bufferWriter.Write(textValue);
                                    break;

                                default:
                                    throw new ArgumentException($"unsupported type on {col.Name}.{colType} row: {rowLine}");
                                }
                                if (throwException)
                                {
                                    throw new ArgumentException($"unable to cast: {textValue} on {col.Name}.{colType} row: {rowLine}");
                                }
                            }
                            //shift right column Bit until it reaches 0 -the last column rightmost
                            columnBit >>= 1;
                        }
                        //write true binary record
                        var flagsBuffer = BitConverter.GetBytes(flags);
                        binWriter.Write(flagsBuffer, 0, table.RowMaskLength);

                        //write non-null records
                        var recBinary = stream.ToArray();
                        binWriter.Write(recBinary, 0, recBinary.Length);

                        //update binary position for next row
                        binaryPosition += table.RowMaskLength + recBinary.Length;
                    }
                }
                ;

                //ellapsed time
                sw.Stop();
                Console.WriteLine("ellapsed {0} ms", sw.ElapsedMilliseconds);
                //var timespan = DateTime.Now - startTime;
                //var ellapsed = $"  ellapsed: {timespan}";
                //Console.WriteLine(ellapsed);

                //store/update line count
                table.Rows        = rowLine;
                table.Pager.Count = pageCount;
                table.Pager.File  = pagerName;

                //save last page
                if (table.Generate)
                {
                    csvPagerwriter.WriteLine(
                        $"{prevGlobalRow}|{prevPosition}|{calculatePageSize(prevPosition, binaryPosition)}");
                }

                Console.WriteLine($"  ({rowLine}) line(s) processed.");

                //close all index writers, and delete temporary files
                indexesArray?.ToList().ForEach(index =>
                {
                    index.writer.Dispose();
                });
                binWriter?.Dispose();
            }
            if (!table.Generate)
            {
                Console.WriteLine($"  indexers were not generated!");
            }
        }