示例#1
0
        public int Read(ImportFileOptions importTask, ISqlWriter sqlWriter)
        {
            Log.Debug($"ReadCsv for '{importTask.file}'");

            Stopwatch _timer = new Stopwatch();

            _timer.Start();

            List <List <string> > batchLineFields = new List <List <string> >();
            var fileInfo = new System.IO.FileInfo(importTask.file);

            if (!System.IO.File.Exists(importTask.file))
            {
                Log.Error($"ReadCSV : '{importTask.file}' not exists!");
                return(-1);
            }

            var firstLine = ReadFirstLine(fileInfo);

            Char delimiter = string.IsNullOrWhiteSpace(importTask.delimiter) ? GuessDelimeter(firstLine) : importTask.delimiter.First <char>();
            //Char quotingCharacter = '\0'; // no quoting-character;
            Char quotingCharacter = string.IsNullOrWhiteSpace(importTask.quoting) ? GuessquotingCharacter(firstLine) : importTask.quoting.First <char>();
            var  rowCounter       = 0;

            try
            {
                using (var reader = new System.IO.StreamReader(fileInfo.FullName, Encoding.Default))
                {
                    Char escapeCharacter = quotingCharacter;
                    using (var csv = new CsvReader(reader, false, delimiter, quotingCharacter, escapeCharacter, '\0', ValueTrimmingOptions.All))
                    {
                        csv.DefaultParseErrorAction = ParseErrorAction.ThrowException;
                        csv.SkipEmptyLines          = true;

                        var headers = ReadHeaders(csv);
                        Log.Debug("Read:Headers : " + string.Join(",", headers));
                        sqlWriter.Init(importTask, headers);

                        if (!sqlWriter.EnsureFileIsUnique(fileInfo.Length))
                        {
                            Log.Debug($"Read: Import failed. The File '{importTask.file}' with size:'{fileInfo.Length}' has been allready imported.");
                            sqlWriter.UpdateStatusTable(-1, new TimeSpan(), fileInfo.Length);
                            return(-2);
                        }

                        do
                        {
                            // Read a pies of CSV
                            batchLineFields = ReadNextBatch(csv, importTask.batchSize);
                            if (batchLineFields.Count == 0)
                            {
                                break;
                            }
                            Log.Debug($"Read next batch : {batchLineFields.Count} entries start at {rowCounter} to {batchLineFields.Count + rowCounter}");

                            //Create & Write SQL
                            sqlWriter.Write(batchLineFields);

                            rowCounter += batchLineFields.Count;
                        } while (batchLineFields.Count > 0);
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Error($"ReadCsv catch exception : {ex.Message}");
                return(-1);
            }

            _timer.Stop();

            var timeSpan = TimeSpan.FromMilliseconds(_timer.ElapsedMilliseconds);

            Log.Debug($"Import from '{importTask.file}' lenght:'{fileInfo.Length}' takes {_timer.ElapsedMilliseconds / 1000.0} seconds or {timeSpan.ToString("c")} of time. {rowCounter} rows inserted.");
            sqlWriter.UpdateStatusTable(rowCounter, timeSpan, fileInfo.Length);

            sqlWriter.ExecuteAdditionalSql();

            return(rowCounter);
        }