ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = GetName(context);
file = context.ResolveFile(file, ResolveFileMode.Output);
context.OutputMessage("Writing file " + Path.GetFullPath(file));
return new CdlFileWriter(file, rowFormat);
}
public ICdlWriter CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = context.ResolveFile(context.Replace(File), ResolveFileMode.Output);
var fw = new StreamWriter(file);
var provider = GetConnectionProvider(context);
return new SqlFileWriter(fw, provider.Factory);
}
public TableWriter(IShellContext context, IConnectionProvider connection, NameWithSchema name, TableInfo inputRowFormat, CopyTableTargetOptions options, TableInfo destinationTableOverride = null, LinkedDatabaseInfo linkedInfo = null, DataFormatSettings sourceDataFormat = null)
{
_connectionProvider = connection;
_linkedInfo = linkedInfo;
_name = name;
_inputRowFormat = inputRowFormat;
_queue = new CdlDataQueue(inputRowFormat);
_context = context;
_inserter = connection.Factory.CreateBulkInserter();
_inserter.SourceDataFormat = sourceDataFormat;
_connection = _connectionProvider.Connect();
_inserter.Connection = _connection;
_inserter.Factory = connection.Factory;
_inserter.LinkedInfo = _linkedInfo;
var db = context.GetDatabaseStructure(connection.ProviderString);
_inserter.DestinationTable = destinationTableOverride ?? db.FindTableLike(_name.Schema, _name.Name);
_inserter.CopyOptions = options;
_inserter.Log += _inserter_Log;
_thread = new Thread(Run);
_thread.Start();
}
public BulkInserterBase(ILogger <BulkInserterBase> logger)
{
_logger = logger;
BatchSize = 100;
CopyOptions = new CopyTableTargetOptions();
}
public BulkInserterBase()
{
BatchSize = 100;
CopyOptions = new CopyTableTargetOptions();
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = context.ResolveFile(context.Replace(Name), ResolveFileMode.Output);
context.OutputMessage("Writing file " + Path.GetFullPath(file));
var dbf = new SocialExplorer.IO.FastDBF.DbfFile(Encoding);
if (File.Exists(file)) File.Delete(file);
dbf.Create(file);
foreach (var col in rowFormat.Columns)
{
DbfColumn.DbfColumnType type;
int len = 0, scale = 0;
switch (col.CommonType.Code)
{
case DbTypeCode.Array:
case DbTypeCode.Generic:
case DbTypeCode.Text:
case DbTypeCode.Xml:
type = DbfColumn.DbfColumnType.Memo;
break;
case DbTypeCode.Blob:
type = DbfColumn.DbfColumnType.Binary;
break;
case DbTypeCode.Datetime:
var dtype = (DbTypeDatetime) col.CommonType;
if (dtype.SubType == DbDatetimeSubType.Date)
{
type = DbfColumn.DbfColumnType.Date;
}
else
{
type = DbfColumn.DbfColumnType.Character;
len = DateTime.UtcNow.ToString("s").Length;
}
break;
case DbTypeCode.Float:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = DefaultNumericScale;
break;
case DbTypeCode.Int:
if (AllowFoxProInteger)
{
type = DbfColumn.DbfColumnType.Integer;
}
else
{
type = DbfColumn.DbfColumnType.Number;
len = 18;
}
break;
case DbTypeCode.Logical:
type = DbfColumn.DbfColumnType.Boolean;
break;
case DbTypeCode.Numeric:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = ((DbTypeNumeric) col.CommonType).Scale;
break;
case DbTypeCode.String:
var stype = (DbTypeString) col.CommonType;
if (stype.IsBinary)
{
type = DbfColumn.DbfColumnType.Binary;
}
else if (stype.Length <= 254)
{
type = DbfColumn.DbfColumnType.Character;
len = stype.Length;
if (len <= 0) len = DefaultStringLength;
}
else
{
type = DbfColumn.DbfColumnType.Memo;
}
break;
default:
type = DbfColumn.DbfColumnType.Character;
len = DefaultStringLength;
break;
}
dbf.Header.AddColumn(col.Name, type, len, scale);
}
return new DbfWriter(dbf, DataFormat);
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
return CreateTarget(context).CreateWriter(rowFormat, options, context, sourceDataFormat);
}
protected override void DoRun(IShellContext context)
{
ITabularDataSource source;
ITabularDataTarget target;
if (Source != null && SourceExpression != null) throw new Exception("DBSH-00087 CopyTable: Both Source and SourceExpression are set");
if (Source == null && SourceExpression == null) throw new Exception("DBSH-00088 CopyTable: None Source and SourceExpression are set");
if (Target != null && TargetExpression != null) throw new Exception("DBSH-00089 CopyTable: Both Target and TargetExpression are set");
if (Target == null && TargetExpression == null) throw new Exception("DBSH-00090 CopyTable: None Target and TargetExpression are set");
if (SourceExpression != null)
{
source = (ITabularDataSource) context.Evaluate(SourceExpression);
}
else
{
source = Source;
}
if (TargetExpression != null)
{
target = (ITabularDataTarget) context.Evaluate(TargetExpression);
}
else
{
target = Target;
}
var options = new CopyTableTargetOptions
{
TruncateBeforeCopy = CleanTarget,
TargetMapMode = TargetMapMode,
AllowBulkCopy = AllowBulkCopy,
};
var table = source.GetRowFormat(context);
_log.InfoFormat("Copy table data {0}=>{1}", Source.ToStringCtx(context), Target.ToStringCtx(context));
context.OutputMessage(String.Format("Copy table data {0}=>{1}", Source.ToStringCtx(context), Target.ToStringCtx(context)));
var transformedInputTable = table;
var counts = new List<int>();
if (ColumnMap.Count > 0)
{
transformedInputTable = new TableInfo(null);
foreach (var mapItem in ColumnMap)
{
var newCols = mapItem.GetOutputColumns(table, context);
counts.Add(newCols.Length);
transformedInputTable.Columns.AddRange(newCols);
}
}
using (var reader = source.CreateReader(context))
{
using (var writer = target.CreateWriter(transformedInputTable, options, context, source.GetSourceFormat(context)))
{
int rowNumber = 0;
while (reader.Read())
{
if (ColumnMap.Count > 0)
{
var outputRecord = new ArrayDataRecord(transformedInputTable);
int columnIndex = 0;
for (int i = 0; i < ColumnMap.Count; i++)
{
var map = ColumnMap[i];
int count = counts[i];
for (int j = 0; j < count; j++, columnIndex++)
{
outputRecord.SeekValue(columnIndex);
map.ProcessMapping(j, rowNumber, reader, outputRecord, context);
}
}
writer.Write(outputRecord);
}
else
{
writer.Write(reader);
}
rowNumber++;
}
}
}
}
public ICdlWriter CreateWriter(TableInfo inputRowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
var connection = GetConnectionProvider(context);
using (var conn = connection.Connect())
{
var db = new DatabaseInfo();
db.LinkedInfo = LinkedInfo;
var tbl = inputRowFormat.CloneTable(db);
tbl.FullName = GetFullName(context);
foreach (var col in tbl.Columns) col.AutoIncrement = false;
tbl.ForeignKeys.Clear();
if (tbl.PrimaryKey != null) tbl.PrimaryKey.ConstraintName = null;
tbl.AfterLoadLink();
if (IdentityColumn != null)
{
var col = new ColumnInfo(tbl);
col.Name = IdentityColumn;
col.DataType = "int";
col.AutoIncrement = true;
col.NotNull = true;
var pk = new PrimaryKeyInfo(tbl);
pk.Columns.Add(new ColumnReference {RefColumn = col});
pk.ConstraintName = "PK_" + tbl.Name;
tbl.PrimaryKey = pk;
tbl.Columns.Insert(0, col);
}
//var sw = new StringWriter();
var so = new ConnectionSqlOutputStream(conn, null, connection.Factory.CreateDialect());
var dmp = connection.Factory.CreateDumper(so, new SqlFormatProperties());
if (DropIfExists) dmp.DropTable(tbl, true);
bool useExistingTable = false;
if (UseIfExists)
{
var ts = context.GetDatabaseStructure(connection.ProviderString);
useExistingTable = ts.FindTableLike(tbl.FullName.Schema, tbl.FullName.Name) != null;
}
if (!useExistingTable)
{
tbl.Columns.ForEach(x => x.EnsureDataType(connection.Factory.CreateSqlTypeProvider()));
dmp.CreateTable(tbl);
}
//using (var cmd = conn.CreateCommand())
//{
// cmd.CommandText = sw.ToString();
// cmd.ExecuteNonQuery();
//}
return new TableWriter(context, connection, GetFullName(context), inputRowFormat, options, useExistingTable ? null : tbl, LinkedInfo, sourceDataFormat);
}
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = context.ResolveFile(GetName(context), ResolveFileMode.Output);
context.OutputMessage("Writing file " + Path.GetFullPath(file));
//var fs = System.IO.File.OpenWrite(file);
var fw = new StreamWriter(file, false, Encoding);
var writer = new CsvWriter(fw, Delimiter, Quote, Escape, Comment, QuotingMode, EndOfLine, DataFormat);
if (HasHeaders)
{
writer.WriteRow(rowFormat.Columns.Select(c => c.Name));
}
return writer;
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo inputRowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
return new TableWriter(context, GetConnectionProvider(context), GetFullName(context), inputRowFormat, options, StructureOverride, LinkedInfo, sourceDataFormat);
}
public ICdlWriter CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
return GetModel(context).CreateWriter(rowFormat, context.Replace(SheetName));
}
