public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts)
{
var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName;
bool autoinc = queue.GetRowFormat.FindAutoIncrementColumn() != null;
if (autoinc)
{
m_dmp.AllowIdentityInsert(table.FullName, true);
}
try
{
if (Cfg != null && Cfg.JoinInserts)
{
int rowsInBatch = 0;
while (!queue.IsEof)
{
if (Cfg.BatchLimit > 0 && rowsInBatch >= Cfg.BatchLimit)
{
m_dmp.EndCommand();
rowsInBatch = 0;
}
if (rowsInBatch > 0)
{
m_dmp.Put(";\n");
}
IBedRecord row = queue.GetRecord();
m_dmp.Put("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row);
rowsInBatch++;
}
if (rowsInBatch > 0)
{
m_dmp.EndCommand();
}
}
else
{
while (!queue.IsEof)
{
IBedRecord row = queue.GetRecord();
m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row);
}
}
}
finally
{
queue.CloseReading();
}
if (autoinc)
{
m_dmp.AllowIdentityInsert(table.FullName, false);
}
}
public virtual void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts)
{
var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName;
bool autoinc = queue.GetRowFormat.FindAutoIncrementColumn() != null;
if (autoinc)
{
m_dmp.AllowIdentityInsert(table.FullName, true);
}
try
{
while (!queue.IsEof)
{
IBedRecord row = queue.GetRecord();
m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row);
}
}
finally
{
queue.CloseReading();
}
if (autoinc)
{
m_dmp.AllowIdentityInsert(table.FullName, false);
}
}
public static void WriteQueueToXml(IDataQueue queue, Stream fw, string dbName, string tableName)
{
DataTable table = ConnTools.DataTableFromStructure(queue.GetRowFormat);
table.TableName = tableName;
List <string> fldnames = new List <string>();
foreach (IColumnStructure col in queue.GetRowFormat.Columns)
{
fldnames.Add(col.ColumnName);
}
XmlWriter xw = XmlTextWriter.Create(fw, new XmlWriterSettings {
Encoding = Encoding.UTF8, CheckCharacters = false
});
//XmlTextWriter xw = new XmlTextWriter(fw, Encoding.UTF8);
//xw.Settings = new XmlWriterSettings { CheckCharacters = false };
xw.WriteStartDocument();
xw.WriteStartElement(dbName);
// mono has bug in writing schema
if (!Core.IsMono)
{
table.WriteXmlSchema(xw);
}
List <string> ids = new List <string>();
foreach (IColumnStructure col in queue.GetRowFormat.Columns)
{
ids.Add(XmlTool.NormalizeIdentifier(col.ColumnName));
}
try
{
while (!queue.IsEof)
{
IBedRecord row = queue.GetRecord();
xw.WriteStartElement(tableName);
for (int i = 0; i < row.FieldCount; i++)
{
row.ReadValue(i);
var type = row.GetFieldType();
if (type == TypeStorage.Null)
{
continue;
}
xw.WriteElementString(ids[i], XmlTool.ObjectToString(row.GetValue(i)));
}
xw.WriteEndElement();
}
}
finally
{
queue.CloseReading();
}
xw.WriteEndElement();
xw.WriteEndDocument();
xw.Flush();
}
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts)
{
var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName;
try
{
if (Wcfg != null && Wcfg.ExtendedInserts)
{
int rowsInBatch = 0;
while (!queue.IsEof)
{
if (Wcfg.BatchLimit > 0 && rowsInBatch >= Wcfg.BatchLimit)
{
m_dmp.EndCommand();
rowsInBatch = 0;
}
if (rowsInBatch == 0)
{
m_dmp.Put("^insert ^into %f (%,i) ^values\n", table, colnames);
}
else
{
m_dmp.Put(",\n");
}
IBedRecord row = queue.GetRecord();
m_dmp.Put("(%,v)", row);
rowsInBatch++;
}
if (rowsInBatch > 0)
{
m_dmp.EndCommand();
}
}
else
{
while (!queue.IsEof)
{
IBedRecord row = queue.GetRecord();
m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row);
}
}
}
finally
{
queue.CloseReading();
}
}
protected override void RunBulkCopy(IDataQueue queue)
{
int okRowCount = 0, failRowCount = 0;
List <string> insertErrors = new List <string>();
ITableStructure dst = queue.GetRowFormat;
var conn = (NpgsqlConnection)Connection.SystemConnection;
NpgsqlCommand command = new NpgsqlCommand(Connection.Dialect.GenerateScript(d => d.Put("^copy %f (%,i) ^from ^stdin", DestinationTable.FullName, from c in dst.Columns select c.ColumnName)), conn);
NpgsqlCopyIn cin = new NpgsqlCopyIn(command, conn);
try
{
cin.Start();
var fw = new BinaryWriter(cin.CopyStream);
while (!queue.IsEof)
{
IBedRecord rec = queue.GetRecord();
for (int i = 0; i < rec.FieldCount; i++)
{
if (i > 0)
{
fw.Write((byte)'\t');
}
rec.ReadValue(i);
WriteField(rec, fw);
}
fw.Write((byte)'\r');
fw.Write((byte)'\n');
okRowCount++;
}
fw.Flush();
cin.End();
}
catch (Exception err)
{
cin.Cancel("canceled");
ProgressInfo.LogMessageDetail(
"INSERT", DatAdmin.LogLevel.Error,
String.Format("{0}", Texts.Get("s_error_inserting_into_table$table", "table", DestinationTable.FullName)), err.ToString());
throw;
}
if (failRowCount > 0)
{
ProgressInfo.LogMessageDetail(
"INSERT", DatAdmin.LogLevel.Error,
String.Format("{0}, OK:{1}, FAIL:{2}", Texts.Get("s_error_inserting_into_table$table", "table", DestinationTable.FullName), okRowCount, failRowCount),
insertErrors.CreateDelimitedText("\r\n")
);
}
else
{
ProgressInfo.LogMessage("INSERT", DatAdmin.LogLevel.Info, Texts.Get("s_inserted_into_table$table$rows", "table", DestinationTable.FullName, "rows", okRowCount));
}
}
public static void WriteQueueToStream(IDataQueue queue, Stream fw, BedDataStats stats)
{
BinaryWriter bw = new BinaryWriter(fw);
bw.Write(SIGNATURE);
bw.Write((int)2); // version
var ts = queue.GetRowFormat;
bw.Write7BitEncodedInt(ts.Columns.Count);
var pk = ts.FindConstraint <IPrimaryKey>();
foreach (IColumnStructure col in ts.Columns)
{
bw.Write(col.ColumnName);
bw.Write((byte)col.DataType.DefaultStorage);
ColFlags flags = 0;
if (pk != null && pk.Columns.IndexOfIf(c => c.ColumnName == col.ColumnName) >= 0)
{
flags |= ColFlags.ISPK;
}
bw.Write((byte)flags);
}
MemoryStream rowdata = new MemoryStream();
BinaryWriter bwrow = new BinaryWriter(rowdata);
try
{
while (!queue.IsEof)
{
rowdata.Position = 0;
rowdata.SetLength(0);
IBedRecord row = queue.GetRecord();
BedTool.SaveRecord(ts.Columns.Count, row, bwrow);
bw.Write7BitEncodedInt((int)rowdata.Length);
rowdata.WriteTo(bw.BaseStream);
if (stats != null)
{
stats.Rows++;
stats.Bytes += (int)rowdata.Length;
stats.Bytes += 4;
}
}
}
finally
{
queue.CloseReading();
}
// write EOF mark
bw.Write7BitEncodedInt(-1);
}
private void DoFillOnBackground(IDataQueue queue)
{
try
{
m_state = TabularDataViewState.Loading;
//DataTable curbuf = null;
Chunk curchunk = null;
try
{
while (!queue.IsEof)
{
if (curchunk == null)
{
curchunk = new Chunk();
}
IBedRecord rec = queue.GetRecord();
curchunk.SaveRecord(rec);
if (curchunk.Count >= BUFFER_SIZE)
{
FlushChunk(curchunk);
curchunk = null;
}
}
}
finally
{
queue.CloseReading();
}
if (curchunk != null)
{
FlushChunk(curchunk);
}
m_state = TabularDataViewState.Prepared;
}
catch (Exception e)
{
Errors.Report(e);
m_state = TabularDataViewState.Error;
queue.PutError(e);
}
finally
{
queue.CloseWriting();
}
if (LoadedNextData != null)
{
LoadedNextData(this, new LoadedNextDataArgs(m_serializedRows));
}
}
public static IEnumerable <IBedRecord> EnumRows(this IDataQueue queue)
{
try
{
while (!queue.IsEof)
{
yield return(queue.GetRecord());
}
}
finally
{
queue.CloseReading();
}
}
public bool Read()
{
if (m_queue.IsEof)
{
base.m_obj = null;
m_recada.Record = null;
return(false);
}
else
{
m_recada.Record = m_queue.GetRecord();
base.m_obj = m_recada;
ReadedRows++;
return(true);
}
}
protected virtual void DoWrite(IDataQueue queue)
{
var fmt = new BedValueFormatter(new DataFormatSettings());
try
{
int index = 0;
while (!queue.IsEof)
{
var record = queue.GetRecord();
record.ReadValue(0);
fmt.ReadFrom(record);
string name = OutputFileNameTemplate;
name = name.Replace("#FILE#", fmt.GetText());
name = name.Replace("#INDEX#", (index + 1).ToString());
string fn = Path.Combine(OutputDirectory, name);
record.ReadValue(1);
switch (record.GetFieldType())
{
case TypeStorage.String:
using (var tw = new StreamWriter(fn, false, m_encoding))
{
tw.Write(record.GetString());
}
break;
case TypeStorage.ByteArray:
using (var fw = new FileInfo(fn).OpenWrite())
{
var data = record.GetByteArray();
fw.Write(data, 0, data.Length);
}
break;
}
index++;
}
}
finally
{
queue.CloseReading();
}
}
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts)
{
var fmt = new BedValueFormatter(FormatSettings);
ExcelXmlDataStore.WriteHeaders(m_fw, table, table.FullName.ToString());
try
{
while (!queue.IsEof)
{
var record = queue.GetRecord();
ExcelXmlDataStore.WriteRow(m_fw, table, record, fmt);
}
}
finally
{
queue.CloseReading();
}
m_fw.Write("</ss:Table>\n");
m_fw.Write("</Worksheet>\n");
}
protected virtual void DoWrite(IDataQueue queue)
{
var api = StreamApi;
if (api == null)
{
throw new NotImplementedError("DAE-00100");
}
StreamWriter fw = GetOutputStream();
if (fw == null)
{
throw new NotImplementedError("DAE-00101");
}
try
{
ITableStructure ts = queue.GetRowFormat;
object manager = null;
api.WriteStart(fw, ts, ref manager);
try
{
int index = 0;
while (!queue.IsEof)
{
var record = queue.GetRecord();
api.WriteRecord(fw, ts, record, index, manager);
index++;
}
}
finally
{
queue.CloseReading();
}
api.WriteEnd(fw, ts, manager);
}
finally
{
fw.Close();
}
FinalizeBulkCopy();
}
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts)
{
ITableStructure ts = queue.GetRowFormat;
string fn = FileNameTemplate;
fn = fn.Replace("#TABLE#", table.FullName.Name);
fn = fn.Replace("#SCHEMA#", table.FullName.Schema ?? "");
string dir = Path.GetDirectoryName(fn);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var api = DataStore.StreamApi;
using (StreamWriter fw = new StreamWriter(fn))
{
object manager = null;
api.WriteStart(fw, ts, ref manager);
int index = 0;
try
{
while (!queue.IsEof)
{
var record = queue.GetRecord();
api.WriteRecord(fw, ts, record, index, manager);
index++;
}
}
finally
{
queue.CloseReading();
}
api.WriteEnd(fw, ts, manager);
}
}
/// <summary>
/// writes content of data queue into output DBF file
/// </summary>
/// <param name="queue"></param>
protected override void DoWrite(IDataQueue queue)
{
var dbf = new DbfFile();
var formatter = new BedValueFormatter(FormatSettings);
try
{
if (File.Exists(GetWorkingFileName()))
{
File.Delete(GetWorkingFileName());
}
dbf.Create(GetWorkingFileName());
ITableStructure ts = queue.GetRowFormat;
foreach (var col in ts.Columns)
{
DbfColumn.DbfColumnType type;
int len = 0, scale = 0;
switch (col.DataType.Code)
{
case DbTypeCode.Array:
case DbTypeCode.Generic:
case DbTypeCode.Text:
case DbTypeCode.Xml:
type = DbfColumn.DbfColumnType.Memo;
break;
case DbTypeCode.Blob:
type = DbfColumn.DbfColumnType.Binary;
break;
case DbTypeCode.Datetime:
var dtype = (DbTypeDatetime)col.DataType;
if (dtype.SubType == DbDatetimeSubType.Date)
{
type = DbfColumn.DbfColumnType.Date;
}
else
{
type = DbfColumn.DbfColumnType.Character;
len = DateTime.UtcNow.ToString("s").Length;
}
break;
case DbTypeCode.Float:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = DefaultNumericScale;
break;
case DbTypeCode.Int:
if (AllowFoxProInteger)
{
type = DbfColumn.DbfColumnType.Integer;
}
else
{
type = DbfColumn.DbfColumnType.Number;
len = 18;
}
break;
case DbTypeCode.Logical:
type = DbfColumn.DbfColumnType.Boolean;
break;
case DbTypeCode.Numeric:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = ((DbTypeNumeric)col.DataType).Scale;
break;
case DbTypeCode.String:
var stype = (DbTypeString)col.DataType;
if (stype.IsBinary)
{
type = DbfColumn.DbfColumnType.Binary;
}
else if (stype.Length <= 254)
{
type = DbfColumn.DbfColumnType.Character;
len = stype.Length;
if (len <= 0)
{
len = DefaultStringLength;
}
}
else
{
type = DbfColumn.DbfColumnType.Memo;
}
break;
default:
type = DbfColumn.DbfColumnType.Character;
len = DefaultStringLength;
break;
}
dbf.Header.AddColumn(col.ColumnName, type, len, scale);
}
var orec = new DbfRecord(dbf.Header);
while (!queue.IsEof)
{
var record = queue.GetRecord();
orec.Clear();
for (int i = 0; i < ts.Columns.Count; i++)
{
record.ReadValue(i);
formatter.ReadFrom(record);
orec[i] = formatter.GetText();
}
dbf.Write(orec);
}
}
finally
{
dbf.Close();
queue.CloseReading();
}
FinalizeBulkCopy();
}
protected override void RunBulkCopy(IDataQueue queue)
{
int recsInBatch = 0;
int okRowCount = 0, failRowCount = 0;
List <string> insertErrors = new List <string>();
StringWriter buf = null;
ISqlDumper dmp = null;
int maxpacket = Int32.Parse(Connection.SystemConnection.ExecuteScalar("select @@max_allowed_packet").ToString());
using (DbTransaction tran = Connection.SystemConnection.BeginTransaction())
{
try
{
ITableStructure dst = queue.GetRowFormat;
while (!queue.IsEof)
{
IBedRecord rec = queue.GetRecord();
if (recsInBatch == 0)
{
buf = new StringWriter();
dmp = Connection.Dialect.CreateDumper(buf);
dmp.Put("^insert ^into %f (%,i) ^values ", DestinationTable.FullName,
from c in dst.Columns select c.ColumnName);
}
else
{
dmp.Put(", ");
}
dmp.Put("(%,v)", rec);
recsInBatch++;
if (recsInBatch >= BatchSize || buf.GetStringBuilder().Length > maxpacket / 4)
{
string sql = buf.ToString();
int bytes = (int)(Encoding.UTF32.GetByteCount(sql) + 0x200);
try
{
Connection.SystemConnection.ExecuteNonQuery(sql, Connection.Dialect);
okRowCount += recsInBatch;
}
catch (Exception err)
{
failRowCount += recsInBatch;
if (insertErrors.Count < 10)
{
insertErrors.Add(err.Message);
}
}
recsInBatch = 0;
}
}
if (recsInBatch > 0)
{
Connection.SystemConnection.ExecuteNonQuery(buf.ToString(), Connection.Dialect);
okRowCount += recsInBatch;
}
}
catch (Exception err)
{
tran.Rollback();
ProgressInfo.LogMessageDetail(
"INSERT", LogLevel.Error,
String.Format("{0}", Texts.Get("s_error_inserting_into_table$table", "table", DestinationTable.FullName)), err.ToString());
throw;
}
tran.Commit();
if (failRowCount > 0)
{
ProgressInfo.LogMessageDetail(
"INSERT", LogLevel.Error,
String.Format("{0}, OK:{1}, FAIL:{2}", Texts.Get("s_error_inserting_into_table$table", "table", DestinationTable.FullName), okRowCount, failRowCount),
insertErrors.CreateDelimitedText("\r\n")
);
}
else
{
ProgressInfo.LogMessage("INSERT", LogLevel.Info, Texts.Get("s_inserted_into_table$table$rows", "table", DestinationTable.FullName, "rows", okRowCount));
}
}
}
protected virtual void RunInserts(IDataQueue queue)
{
Connection.SystemConnection.SafeChangeDatabase(DatabaseName);
var dda = Connection.GetAnyDDA();
using (DbCommand inscmd = Connection.DbFactory.CreateCommand())
{
List <string> colnames = new List <string>();
List <string> vals = new List <string>();
ITableStructure ts = queue.GetRowFormat;
ITableStructure dst_ts = DestinationTable;
foreach (IColumnStructure col in ts.Columns)
{
vals.Add("{" + colnames.Count.ToString() + "}");
colnames.Add(col.ColumnName);
}
string[] values = new string[colnames.Count];
NameWithSchema table = DestinationTable.FullName;
string insertTemplate = SqlDumper.Format(Connection.Dialect, "^insert ^into %f (%,i) ^values (%,s)", table, colnames, vals);
bool hasident = HasIdentity(queue);
DbTransaction trans = Connection.SystemConnection.BeginTransaction();
inscmd.Connection = Connection.SystemConnection;
inscmd.Transaction = trans;
int okRowCount = 0, failRowCount = 0;
List <string> insertErrors = new List <string>();
try
{
if (hasident)
{
Connection.RunScript(dmp => { dmp.AllowIdentityInsert(table, true); }, trans, ProgressInfo);
}
try
{
int rowcounter = 0;
while (!queue.IsEof)
{
rowcounter++;
IBedRecord row = queue.GetRecord();
for (int i = 0; i < row.FieldCount; i++)
{
row.ReadValue(i);
values[i] = dda.GetSqlLiteral(row);
}
inscmd.CommandText = String.Format(insertTemplate, values);
if (rowcounter > 10000)
{
// next transaction
trans.Commit();
trans.Dispose();
trans = Connection.SystemConnection.BeginTransaction();
inscmd.Transaction = trans;
rowcounter = 0;
}
try
{
inscmd.ExecuteNonQuery();
okRowCount++;
}
catch (Exception err)
{
if (insertErrors.Count < 10)
{
StringBuilder msg = new StringBuilder();
msg.Append(err.Message);
insertErrors.Add(msg.ToString());
}
failRowCount++;
}
}
}
finally
{
if (hasident)
{
Connection.RunScript(dmp => { dmp.AllowIdentityInsert(table, false); }, trans, ProgressInfo);
}
}
trans.Commit();
if (failRowCount > 0)
{
ProgressInfo.LogMessageDetail(
"INSERT", LogLevel.Error,
String.Format("{0}, OK:{1}, FAIL:{2}", Texts.Get("s_error_inserting_into_table$table", "table", DestinationTable.FullName), okRowCount, failRowCount),
insertErrors.CreateDelimitedText("\r\n")
);
}
else
{
ProgressInfo.LogMessage("INSERT", LogLevel.Info, Texts.Get("s_inserted_into_table$table$rows", "table", DestinationTable.FullName, "rows", okRowCount));
}
}
catch (Exception)
{
trans.Rollback();
throw;
}
}
}
