public SmartDataReader(TDataReader dataReader, TypedDataColumnInfo[] destinationColumns,
DataTransformGroup dataTransformGroup = null) : base(dataReader)
{
dataTransformGroup ??= DataTransformGroups.Default;
DataTransformsInDestinationOrder =
destinationColumns
.StuffToArray(p => p.Ordinal)
.Select(destCol => destCol == null ? null : dataTransformGroup(destCol.DataType))
.ToArray();
_mappingInfoLazy = new Lazy <ColumnMappingInfo>(() => dataReader.GetColumnMappings(destinationColumns));
}
public static async Task Upload(SqlConnection sqlc, IDataReader sourceReader, DataUploaderTask task, CancellationToken cancellationToken, string destinationTable, bool useOrdinals, int bulkCopyRowsPerBatch = 5000, DataTransformGroup transformGroup = null, string filterNullColumns = null)
{
long lastCopiedRow = -1;
SmartDataReader <IDataReader> smartReader = null;//.ApplyTransformation(1, row => DataTransforms.TransformExcelDate(row[1]));
sourceReader = sourceReader.CountRows();
try
{
var reader = sourceReader;
if (string.IsNullOrWhiteSpace(filterNullColumns) == false)
{
var cols = filterNullColumns.Split(",");
reader = reader.Where(dataReader =>
{
return(cols.All(col => !string.IsNullOrWhiteSpace(dataReader[col]?.ToString())));
});
}
smartReader = reader.MapToSqlDestination(destinationTable, sqlc, transformGroup);
var progress = new Progress <int>(i =>
{
task.StatusMessage = $"{i} Rows Copied";
});
var drr = smartReader
.NotifyOn(progress, bulkCopyRowsPerBatch)
.CountRows();
await drr.BulkInsertSqlServerAsync(sqlc, destinationTable, new AsyncSqlServerBulkInsertOptions
{
BatchSize = bulkCopyRowsPerBatch,
CancellationToken = cancellationToken,
UseOrdinals = useOrdinals
});
task.Success = true;
task.StatusMessage = $"Finished - {sourceReader.Depth} Rows Read / {drr.Depth} Rows Copied";
}
//csv reader doesn't report depth
catch (Exception ex)
{
if (lastCopiedRow == -1 && sourceReader.Depth <= 0)
{
task.StatusMessage = "No rows were copied. ";
}
else
{
task.StatusMessage = $"{sourceReader.Depth} total rows were read. ";
}
var inners = Try.Get(() => ex.ConcatenateInners(), "");
var readerDiagnostics = Try.Get(() => smartReader?.PrintDiagnostics(), "");
task.StatusMessage += "Exception occurred: " + inners + ". " + readerDiagnostics;
}
}
