public SmartDataReader(TDataReader dataReader, TypedDataColumnInfo[] destinationColumns, DataTransformGroup dataTransformGroup = null) : base(dataReader) { dataTransformGroup ??= DataTransformGroups.Default; DataTransformsInDestinationOrder = destinationColumns .StuffToArray(p => p.Ordinal) .Select(destCol => destCol == null ? null : dataTransformGroup(destCol.DataType)) .ToArray(); _mappingInfoLazy = new Lazy <ColumnMappingInfo>(() => dataReader.GetColumnMappings(destinationColumns)); }
public static async Task Upload(SqlConnection sqlc, IDataReader sourceReader, DataUploaderTask task, CancellationToken cancellationToken, string destinationTable, bool useOrdinals, int bulkCopyRowsPerBatch = 5000, DataTransformGroup transformGroup = null, string filterNullColumns = null) { long lastCopiedRow = -1; SmartDataReader <IDataReader> smartReader = null;//.ApplyTransformation(1, row => DataTransforms.TransformExcelDate(row[1])); sourceReader = sourceReader.CountRows(); try { var reader = sourceReader; if (string.IsNullOrWhiteSpace(filterNullColumns) == false) { var cols = filterNullColumns.Split(","); reader = reader.Where(dataReader => { return(cols.All(col => !string.IsNullOrWhiteSpace(dataReader[col]?.ToString()))); }); } smartReader = reader.MapToSqlDestination(destinationTable, sqlc, transformGroup); var progress = new Progress <int>(i => { task.StatusMessage = $"{i} Rows Copied"; }); var drr = smartReader .NotifyOn(progress, bulkCopyRowsPerBatch) .CountRows(); await drr.BulkInsertSqlServerAsync(sqlc, destinationTable, new AsyncSqlServerBulkInsertOptions { BatchSize = bulkCopyRowsPerBatch, CancellationToken = cancellationToken, UseOrdinals = useOrdinals }); task.Success = true; task.StatusMessage = $"Finished - {sourceReader.Depth} Rows Read / {drr.Depth} Rows Copied"; } //csv reader doesn't report depth catch (Exception ex) { if (lastCopiedRow == -1 && sourceReader.Depth <= 0) { task.StatusMessage = "No rows were copied. "; } else { task.StatusMessage = $"{sourceReader.Depth} total rows were read. "; } var inners = Try.Get(() => ex.ConcatenateInners(), ""); var readerDiagnostics = Try.Get(() => smartReader?.PrintDiagnostics(), ""); task.StatusMessage += "Exception occurred: " + inners + ". " + readerDiagnostics; } }