private List<DataSetInstance> DoLoadRows(ICdlReader reader, NameWithSchema targetTable)
{
var loaded = new List<DataSetInstance>();
var ts = reader.Structure;
var cls = GetClass(targetTable);
var cols = cls.Structure.Columns.ToList();
int[] map = new int[cols.Count];
for (int i = 0; i < cols.Count; i++)
{
map[i] = ts.Columns.IndexOfIf(
c => System.String.Compare(c.Name, cols[i].Name, System.StringComparison.OrdinalIgnoreCase) == 0);
}
while (reader.Read())
{
if (_loadingStopped) return loaded;
object[] newValues = new object[map.Length];
for (int i = 0; i < map.Length; i++)
{
if (map[i] < 0) continue;
reader.ReadValue(map[i]);
newValues[i] = reader.GetValue();
}
var added = cls.AddRecord(newValues);
if (added != null) loaded.Add(added);
}
return loaded;
}
public static IEnumerable <ICdlRecord> EnumRows(this ICdlReader reader)
{
while (reader.Read())
{
yield return(reader);
}
}
public ColumnMapperReader(ICdlReader source, TableInfo outputFormat, List<IColumnMapping> columnMap, List<int> counts, IShellContext context )
: base(outputFormat)
{
_source = source;
_columnMap = columnMap;
_counts = counts;
_context = context;
}
public ColumnMapperReader(ICdlReader source, TableInfo outputFormat, List <IColumnMapping> columnMap, List <int> counts, IShellContext context)
: base(outputFormat)
{
_source = source;
_columnMap = columnMap;
_counts = counts;
_context = context;
}
public static string SafeString(this ICdlReader row, int ord)
{
if (ord < 0)
{
return(null);
}
row.ReadValue(ord);
return(row.GetValue().SafeToString());
}
public InMemoryTable(TableInfo table, ICdlReader reader)
{
Initialize();
m_structure = table.CloneTable();
while (reader.Read())
{
m_rows.Add(new ArrayDataRecord(reader));
}
}
public static CdlTable ToBinaryTable(this ICdlReader reader, int?maximumRecords)
{
//TableInfo ts = reader.GetTableInfo();
CdlTable dt = new CdlTable(reader.Structure);
int allow_recs = maximumRecords != null ? maximumRecords.Value : -1;
while (reader.Read() && (maximumRecords == null || allow_recs > 0))
{
dt.AddRow(reader);
allow_recs--;
}
return(dt);
}
protected bool HasIdentity(ICdlReader reader)
{
var ts = reader.Structure;
var dst_ts = DestinationTable;
var autoinc = dst_ts.FindAutoIncrementColumn();
bool hasident = false;
if (autoinc != null)
{
hasident = _columnMap.GetSourceColumnByTargetIndex(autoinc.ColumnOrder) != null;
}
return(hasident);
}
public virtual void Run(ICdlReader reader)
{
_columnMap = new TargetColumnMap(reader.Structure, DestinationTable, CopyOptions.TargetMapMode);
var toDb = new RecordToDbAdapter(_columnMap, Factory, SourceDataFormat ?? new DataFormatSettings());
var adapter = new CdlReaderToDbAdapter(toDb, reader);
BeforeRun();
if (CopyOptions.AllowBulkCopy)
{
RunBulkCopy(adapter);
}
else
{
RunInserts(adapter);
}
AfterRun();
}
private bool NextRefReader()
{
_currentFileReader = null;
RefObject = null;
if (_currentIndex + 1 >= _files.Length) return false;
_currentIndex++;
using (var childCtx = _context.CreateChildContext())
{
childCtx.SetVariable(_context.Replace(_propertyName), _files[_currentIndex]);
_currentFileReader = _source.CreateReader(childCtx);
RefObject = _currentFileReader;
}
return true;
}
public static double SafeDouble(this ICdlReader row, int ord)
{
string sval = row.SafeString(ord);
if (sval == null)
{
return(0);
}
double res;
if (Double.TryParse(sval, NumberStyles.Any, CultureInfo.InvariantCulture, out res))
{
return(res);
}
return(res);
}
public virtual void Run(ICdlReader reader)
{
_columnMap = new TargetColumnMap(reader.Structure, DestinationTable, CopyOptions.TargetMapMode);
var toDb = new RecordToDbAdapter(_columnMap, Factory, SourceDataFormat ?? new DataFormatSettings());
var adapter = new CdlReaderToDbAdapter(toDb, reader);
BeforeRun();
if (CopyOptions.AllowBulkCopy)
{
RunBulkCopy(adapter);
}
else
{
RunInserts(adapter);
}
AfterRun();
}
private bool NextRefReader()
{
_currentFileReader = null;
RefObject = null;
if (_currentIndex + 1 >= _files.Length)
{
return(false);
}
_currentIndex++;
using (var childCtx = _context.CreateChildContext())
{
childCtx.SetVariable(_context.Replace(_propertyName), _files[_currentIndex]);
_currentFileReader = _source.CreateReader(childCtx);
RefObject = _currentFileReader;
}
return(true);
}
public static void RunForEachRecordAndDispose(this ICdlReader reader, bool allowDirectCall, Action <ICdlRecord, int> func)
{
if (allowDirectCall)
{
try
{
int index = 0;
while (reader.Read())
{
func(reader, index);
index++;
}
}
finally
{
reader.Dispose();
}
}
else
{
List <ArrayDataRecord> cache = new List <ArrayDataRecord>();
try
{
while (reader.Read())
{
cache.Add(new ArrayDataRecord(reader));
}
}
finally
{
reader.Dispose();
}
int index = 0;
foreach (var rec in cache)
{
func(rec, index);
index++;
}
}
}
public void Close()
{
_isClosed = true;
_reader.Dispose();
_reader = null;
}
protected bool HasIdentity(ICdlReader reader)
{
var ts = reader.Structure;
var dst_ts = DestinationTable;
var autoinc = dst_ts.FindAutoIncrementColumn();
bool hasident = false;
if (autoinc != null)
{
hasident = _columnMap.GetSourceColumnByTargetIndex(autoinc.ColumnOrder) != null;
}
return hasident;
}
public static CdlTable ToBinaryTable(this ICdlReader reader)
{
return(ToBinaryTable(reader, null));
}
protected virtual void RunBulkCopy(ICdlReader reader)
{
RunInserts(reader);
}
public void Close()
{
_isClosed = true;
_reader.Dispose();
_reader = null;
}
public CdlReaderToDbAdapter(IRecordToDbAdapter recordAdapter, ICdlReader sourceReader)
{
_sourceReader = sourceReader;
_recordAdapter = recordAdapter;
RefObject = sourceReader;
}
public CdlReaderToDbAdapter(IRecordToDbAdapter recordAdapter, ICdlReader sourceReader)
{
_sourceReader = sourceReader;
_recordAdapter = recordAdapter;
RefObject = sourceReader;
}
protected virtual void RunInserts(ICdlReader reader)
{
//Connection.SystemConnection.SafeChangeDatabase(DatabaseName);
var dda = Connection.GetFactory(ServiceProvider).CreateDataAdapter();
using (DbCommand inscmd = Connection.CreateCommand())
{
List <string> colnames = new List <string>();
List <string> vals = new List <string>();
foreach (var colIndexes in _columnMap.Items)
{
vals.Add("{" + colnames.Count.ToString() + "}");
colnames.Add(DestinationTable.Columns[colIndexes.Target].Name);
}
string[] values = new string[colnames.Count];
NameWithSchema table = DestinationTable.FullName;
string insertTemplate = SqlDumper.Format(Connection.GetFactory(ServiceProvider), "^insert ^into %f (%,i) ^values (%,s)", table, colnames, vals);
bool hasident = HasIdentity(reader);
DbTransaction trans = Connection.BeginTransaction();
inscmd.Transaction = trans;
int okRowCount = 0, failRowCount = 0;
List <string> insertErrors = new List <string>();
try
{
if (hasident)
{
Connection.RunScript(ServiceProvider, dmp => { dmp.AllowIdentityInsert(table, true); }, trans);
}
try
{
int rowcounter = 0;
while (reader.Read())
{
rowcounter++;
var row = reader;
for (int i = 0; i < _columnMap.Items.Count; i++)
{
row.ReadValue(_columnMap.Items[i].Source);
values[i] = dda.GetSqlLiteral(row, new DbTypeString());
}
inscmd.CommandText = String.Format(insertTemplate, values);
if (rowcounter > 10000)
{
// next transaction
trans.Commit();
trans.Dispose();
trans = Connection.BeginTransaction();
inscmd.Transaction = trans;
rowcounter = 0;
}
try
{
inscmd.ExecuteNonQuery();
okRowCount++;
}
catch (Exception err)
{
if (insertErrors.Count < 10)
{
StringBuilder msg = new StringBuilder();
msg.Append(err.Message);
insertErrors.Add(msg.ToString());
}
failRowCount++;
}
}
}
finally
{
if (hasident)
{
Connection.RunScript(ServiceProvider, dmp => { dmp.AllowIdentityInsert(table, false); }, trans);
}
}
trans.Commit();
if (failRowCount > 0)
{
LogError($"DBSH-00199 Error inserting into table {DestinationTable}, correct inserts {okRowCount}, failed inserts {failRowCount}");
LogError(insertErrors.CreateDelimitedText("\n"));
}
else
{
LogInfo($"{okRowCount} rows successfully inserted into table {DestinationTable}");
}
}
catch (Exception)
{
trans.Rollback();
throw;
}
}
}
protected virtual void RunInserts(ICdlReader reader)
{
//Connection.SystemConnection.SafeChangeDatabase(DatabaseName);
var dda = Connection.GetFactory().CreateDataAdapter();
using (DbCommand inscmd = Connection.CreateCommand())
{
List<string> colnames = new List<string>();
List<string> vals = new List<string>();
foreach (var colIndexes in _columnMap.Items)
{
vals.Add("{" + colnames.Count.ToString() + "}");
colnames.Add(DestinationTable.Columns[colIndexes.Target].Name);
}
string[] values = new string[colnames.Count];
NameWithSchema table = DestinationTable.FullName;
string insertTemplate = SqlDumper.Format(Connection.GetFactory(), "^insert ^into %f (%,i) ^values (%,s)", table, colnames, vals);
bool hasident = HasIdentity(reader);
DbTransaction trans = Connection.BeginTransaction();
inscmd.Transaction = trans;
int okRowCount = 0, failRowCount = 0;
List<string> insertErrors = new List<string>();
try
{
if (hasident) Connection.RunScript(dmp => { dmp.AllowIdentityInsert(table, true); }, trans);
try
{
int rowcounter = 0;
while (reader.Read())
{
rowcounter++;
var row = reader;
for (int i = 0; i < _columnMap.Items.Count; i++)
{
row.ReadValue(_columnMap.Items[i].Source);
values[i] = dda.GetSqlLiteral(row, new DbTypeString());
}
inscmd.CommandText = String.Format(insertTemplate, values);
if (rowcounter > 10000)
{
// next transaction
trans.Commit();
trans.Dispose();
trans = Connection.BeginTransaction();
inscmd.Transaction = trans;
rowcounter = 0;
}
try
{
inscmd.ExecuteNonQuery();
okRowCount++;
}
catch (Exception err)
{
if (insertErrors.Count < 10)
{
StringBuilder msg = new StringBuilder();
msg.Append(err.Message);
insertErrors.Add(msg.ToString());
}
failRowCount++;
}
}
}
finally
{
if (hasident) Connection.RunScript(dmp => { dmp.AllowIdentityInsert(table, false); }, trans);
}
trans.Commit();
if (failRowCount > 0)
{
LogError($"DBSH-00199 Error inserting into table {DestinationTable}, correct inserts {okRowCount}, failed inserts {failRowCount}");
LogError(insertErrors.CreateDelimitedText("\n"));
}
else
{
LogInfo($"{okRowCount} rows successfully inserted into table {DestinationTable}");
}
}
catch (Exception)
{
trans.Rollback();
throw;
}
}
}
protected virtual void RunBulkCopy(ICdlReader reader)
{
RunInserts(reader);
}
public static string SafeString(this ICdlReader row, string field)
{
int ord = row.Structure.Columns.IndexOfIf(x => x.Name == field);
return(row.SafeString(ord));
}
