public CsvReader(TableInfo structure, LumenWorks.Framework.IO.Csv.CsvReader reader, DataFormatSettings dataFormat)
: base(structure)
{
_reader = reader;
_array = new string[structure.ColumnCount];
_dataFormat = dataFormat ?? new DataFormatSettings();
}
public TargetColumnMap(TableInfo source, TableInfo target, TargetColumnMapMode mode)
{
Source = source;
Target = target;
MapMode = mode;
Initialize();
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = GetName(context);
file = context.ResolveFile(file, ResolveFileMode.Output);
context.OutputMessage("Writing file " + Path.GetFullPath(file));
return new CdlFileWriter(file, rowFormat);
}
public ExcelReader(TableInfo structure, Worksheet worksheet)
: base(structure)
{
_worksheet = worksheet;
_array = new string[structure.ColumnCount];
_usedRange = _worksheet.UsedRange;
}
public ICdlWriter CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = context.ResolveFile(context.Replace(File), ResolveFileMode.Output);
var fw = new StreamWriter(file);
var provider = GetConnectionProvider(context);
return new SqlFileWriter(fw, provider.Factory);
}
public DbfReader(TableInfo structure, SocialExplorer.IO.FastDBF.DbfFile dbf)
: base(structure)
{
_dbf = dbf;
_array = new string[structure.ColumnCount];
_irec = new DbfRecord(_dbf.Header);
}
public ColumnMapperReader(ICdlReader source, TableInfo outputFormat, List<IColumnMapping> columnMap, List<int> counts, IShellContext context )
: base(outputFormat)
{
_source = source;
_columnMap = columnMap;
_counts = counts;
_context = context;
}
public override void DropTable(TableInfo obj, bool testIfExists)
{
if (testIfExists)
{
Put("IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'%f') AND type in (N'U'))&n", obj.FullName);
}
base.DropTable(obj, testIfExists);
}
internal CdlRow(CdlTable table, ICdlRecord original, CdlRowState initialState, TableInfo structure)
{
m_table = table;
m_fields = new FieldRec[m_table.Structure.Columns.Count];
m_original = original;
m_structure = structure;
RowState = initialState;
}
public XmlDocumentReader(XmlDocument doc, TableInfo rowFormat, List<XmlReadInstructions> instructions)
: base(rowFormat)
{
_doc = doc;
_rowFormat = rowFormat;
_instructions = instructions;
_enumerator = GetRecords().GetEnumerator();
}
public void AfterLoadLink(TableInfo table)
{
if (_refColumnName != null)
{
RefColumn = table.Columns.Find(c => c.Name == _refColumnName);
_refColumnName = null;
}
}
public TargetEntitySqlModel(DataSyncSqlModel dataSyncSqlModel, Target dbsh, IShellContext context)
{
this._dataSyncSqlModel = dataSyncSqlModel;
this._dbsh = dbsh;
TargetTable = new NameWithSchema(context.Replace(dbsh.TableSchema), context.Replace(dbsh.TableName));
string findSchema = dbsh.TableSchema;
if (findSchema != null && findSchema.StartsWith(NameWithSchema.NoQuotePrefix)) findSchema = null;
Structure = dataSyncSqlModel.TargetStructure.FindTableLike(findSchema, TargetTable.Name);
SqlAlias = _dbsh.Alias ?? "dst_" + _dataSyncSqlModel.Entities.Count;
foreach (var col in dbsh.Columns)
{
var targetCol = new TargetNoRefColumnSqlModel(col, FindColumnInfo(col.Name));
TargetColumns.Add(targetCol);
foreach (string alias in ExtractColumnSources(col))
{
SourceColumnSqlModel source = null;
if (dataSyncSqlModel.SourceGraphModel == null)
{
// flat sync
if (!String.IsNullOrEmpty(dbsh.PrimarySource))
{
var tableSource = DataSync.FlatSources.FirstOrDefault(x => x.Match(Dbsh.PrimarySource));
if (tableSource != null)
{
source = tableSource.Columns.FirstOrDefault(x => x.Alias == alias);
}
}
}
else
{
source = dataSyncSqlModel.SourceGraphModel[alias];
//targetCol.Sources.Add(source);
}
RequiredSourceColumns.Add(source);
if (col.IsKey) KeySourceColumns.Add(source);
}
}
if (!String.IsNullOrEmpty(_dbsh.Connection))
{
var ctxConn = new NormalizedDatabaseConnectionInfo(new DatabaseConnectionInfoHolder { ProviderString = context.GetDefaultConnection() });
var tableConn = new NormalizedDatabaseConnectionInfo(new DatabaseConnectionInfoHolder { ProviderString = context.Replace(_dbsh.Connection), LinkedInfo = _dbsh.LinkedInfo });
if (ctxConn != tableConn)
{
if (ctxConn.ServerConnectionString == tableConn.ServerConnectionString)
{
TargetLinkedInfo = tableConn.GetLinkedInfo();
}
else
{
throw new IncorrectRdsDefinitionException($"DBSH-00000 RDS target must be reachable by database or linked server: ({TargetTable})");
}
}
}
}
public static DataTable DataTableFromStructure(TableInfo tableStruct)
{
DataTable table = new DataTable();
foreach (var col in tableStruct.Columns)
{
DataColumn column = new DataColumn(col.Name, col.CommonType.DotNetType);
table.Columns.Add(column);
}
return table;
}
public static TableInfo GetTableInfo(this DataColumnCollection columns)
{
TableInfo res = new TableInfo(null);
foreach (DataColumn col in columns.SortedByKey<DataColumn, int>(col => col.Ordinal))
{
var commonType = TypeTool.GetCommonType(col.DataType);
var colInfo = res.AddColumn(col.ColumnName, commonType.ToString(), TypeTool.GetCommonType(col.DataType));
colInfo.NotNull = !col.AllowDBNull;
}
return res;
}
public CdlFileWriter(string file, TableInfo table)
{
var fw = new FileInfo(file).OpenWrite();
_bw = new BinaryWriter(fw);
var doc = XmlTool.CreateDocument("Table");
var tcopy = table.CloneTable();
tcopy.ForeignKeys.Clear();
if (tcopy.PrimaryKey != null) tcopy.PrimaryKey.ConstraintName = null;
tcopy.SaveToXml(doc.DocumentElement);
_bw.Write(doc.GetPackedDocumentXml());
}
private void OpenRead(out TableInfo table, out BinaryReader br, IShellContext context)
{
string file = GetName(context);
file = context.ResolveFile(file, ResolveFileMode.Input);
var fr = new FileInfo(file).OpenRead();
br = new BinaryReader(fr);
string s = br.ReadString();
var doc = new XmlDocument();
doc.LoadXml(s);
table = new TableInfo(null);
table.LoadFromXml(doc.DocumentElement);
table.AfterLoadLink();
}
public ArrayDataRecord(ICdlRecord record, int[] colindexes, TableInfo changedStructure)
{
if (colindexes.Length != changedStructure.Columns.Count) throw new InternalError("DBSH-00050 ArrayDataRecord(): colnames.count != colindexes.count");
_values = new object[colindexes.Length];
for (int i = 0; i < colindexes.Length; i++)
{
if (colindexes[i] >= 0)
{
_values[i] = record.GetValue(colindexes[i]);
}
}
_structure = changedStructure;
}
public static TableInfo GetStructure(bool addFileIdentifier)
{
var res = new TableInfo(null);
res.AddColumn("_ShapeId_", "int", new DbTypeInt());
res.AddColumn("_PartId_", "int", new DbTypeInt());
res.AddColumn("_VertexId_", "int", new DbTypeInt());
res.AddColumn("_X_", "float", new DbTypeFloat());
res.AddColumn("_Y_", "float", new DbTypeFloat());
if (addFileIdentifier)
{
res.AddColumn("_File_", "nvarchar(250)", new DbTypeString());
}
return res;
}
private Worksheet CreateSheet(TableInfo rowFormat, string sheetName)
{
Worksheet sheet;
if (_usedFirstSheet) sheet = _workbook.Sheets.Add();
else sheet = _workbook.Sheets[1];
sheet.Name = sheetName;
_usedFirstSheet = true;
for (int i = 0; i < rowFormat.ColumnCount; i++)
{
((Range) sheet.Cells[1, i + 1]).Value2 = rowFormat.Columns[i].Name;
}
return sheet;
}
public SqliteStorage(TableInfo table)
{
_file = Path.GetTempFileName();
_table = table;
_conn = new SQLiteConnection("Synchronous=Full;Data Source=" + _file);
_conn.Open();
string sql = String.Format("create table {0} ({1})", TABLE_NAME, ColumnsText);
_conn.ExecuteNonQuery(sql);
lock (_storageDirectory)
{
_storageDirectory[_file] = this;
}
}
private TableInfo GetRowFormat(List<int> counts, IShellContext context)
{
var table = Source.GetRowFormat(context);
var targetTable = table;
targetTable = new TableInfo(null);
foreach (var mapItem in ColumnMap)
{
var newCols = mapItem.GetOutputColumns(table, context);
counts.Add(newCols.Length);
targetTable.Columns.AddRange(newCols);
}
return targetTable;
}
TableInfo GetStructure(SocialExplorer.IO.FastDBF.DbfFile dbf)
{
var res = new TableInfo(null);
//output column names
for (int i = 0; i < dbf.Header.ColumnCount; i++)
{
DbTypeBase type;
// convert DBF type to DA type
switch (dbf.Header[i].ColumnType)
{
case DbfColumn.DbfColumnType.Binary:
type = new DbTypeBlob();
break;
case DbfColumn.DbfColumnType.Boolean:
type = new DbTypeLogical();
break;
case DbfColumn.DbfColumnType.Date:
type = new DbTypeDatetime {SubType = DbDatetimeSubType.Date};
break;
case DbfColumn.DbfColumnType.Character:
type = new DbTypeString {Length = dbf.Header[i].Length};
break;
case DbfColumn.DbfColumnType.Integer:
type = new DbTypeInt();
break;
case DbfColumn.DbfColumnType.Memo:
type = new DbTypeText();
break;
case DbfColumn.DbfColumnType.Number:
type = new DbTypeNumeric
{
Precision = dbf.Header[i].Length,
Scale = dbf.Header[i].DecimalCount,
};
break;
default:
type = new DbTypeString();
break;
}
var col = new ColumnInfo(res);
col.Name = dbf.Header[i].Name;
col.CommonType = type;
//col.FillTypeFromCommonType();
res.Columns.Add(col);
}
return res;
}
public TableInfo ToTableInfo(bool includeHiddenColumns = false)
{
var res = new TableInfo(new DatabaseInfo());
var pk = new PrimaryKeyInfo(res);
var tableNames = Columns.Select(x => x.BaseTableName).Where(x => x != null).Distinct().ToList();
var schemaNames = Columns.Select(x => x.BaseSchemaName).Where(x => x != null).Distinct().ToList();
if (tableNames.Count == 1 && schemaNames.Count <= 1)
{
res.FullName = new NameWithSchema(schemaNames.FirstOrDefault(), tableNames.Single());
}
foreach (var column in Columns)
{
if (column.IsHidden && !includeHiddenColumns) continue;
var col = new ColumnInfo(res)
{
Name = column.Name,
NotNull = column.NotNull,
CommonType = column.CommonType.Clone(),
DataType = column.DataType,
AutoIncrement = column.AutoIncrement,
PrimaryKey = column.IsKey,
};
if (col.CommonType is DbTypeString) col.Length = column.Size;
if (column.AutoIncrement && col.CommonType != null)
{
col.CommonType.SetAutoincrement(true);
}
if (column.IsKey && res.FullName != null)
{
pk.Columns.Add(new ColumnReference {RefColumn = col});
}
res.Columns.Add(col);
}
if (pk.Columns.Count > 0) res.PrimaryKey = pk;
return res;
}
public TableInfo CreateColumnSubset(IEnumerable<int> columnSubset)
{
if (columnSubset != null)
{
var ts = new TableInfo(OwnerDatabase);
foreach (int colindex in columnSubset)
{
var src = Columns[colindex];
ts.Columns.Add(new ColumnInfo(ts)
{
Name = src.Name,
DataType = src.DataType,
NotNull = src.NotNull,
CommonType = src.CommonType,
});
}
return ts;
}
return this;
}
protected static void GetValues(DmlfUpdateFieldCollection fields, List<ChangeSetValue> values, TableInfo table, IDialectDataAdapter dda, ICdlValueConvertor converter)
{
var input = new CdlValueHolder();
var output = new CdlValueHolder();
foreach (var col in values)
{
var colinfo = table.FindColumn(col.Column);
if (colinfo == null) continue;
input.ReadFrom(col.Value);
dda.AdaptValue(input, colinfo.CommonType, output, converter);
fields.Add(new DmlfUpdateField
{
TargetColumn = colinfo.Name,
Expr = new DmlfLiteralExpression
{
Value = output.GetValue(),
}
});
}
}
private void ProcessTable(TableInfo table)
{
if (Column != null)
{
var column = table.FindColumn(Column);
if (column != null)
{
column.Properties[Name] = Value;
}
}
if (Columns != null)
{
foreach (var column in table.Columns)
{
if (!Regex.Match(column.Name, Columns).Success) continue;
if (IsIdentity && !column.AutoIncrement) continue;
if (IsPrimaryKey && !column.PrimaryKey) continue;
column.Properties[Name] = Value;
}
}
}
public TableWriter(IShellContext context, IConnectionProvider connection, NameWithSchema name, TableInfo inputRowFormat, CopyTableTargetOptions options, TableInfo destinationTableOverride = null, LinkedDatabaseInfo linkedInfo = null, DataFormatSettings sourceDataFormat = null)
{
_connectionProvider = connection;
_linkedInfo = linkedInfo;
_name = name;
_inputRowFormat = inputRowFormat;
_queue = new CdlDataQueue(inputRowFormat);
_context = context;
_inserter = connection.Factory.CreateBulkInserter();
_inserter.SourceDataFormat = sourceDataFormat;
_connection = _connectionProvider.Connect();
_inserter.Connection = _connection;
_inserter.Factory = connection.Factory;
_inserter.LinkedInfo = _linkedInfo;
var db = context.GetDatabaseStructure(connection.ProviderString);
_inserter.DestinationTable = destinationTableOverride ?? db.FindTableLike(_name.Schema, _name.Name);
_inserter.CopyOptions = options;
_inserter.Log += _inserter_Log;
_thread = new Thread(Run);
_thread.Start();
}
ColumnInfo[] IColumnMapping.GetOutputColumns(TableInfo inputTable, IShellContext context)
{
var column = new ColumnInfo(new TableInfo(null)) {CommonType = new DbTypeString(), Name = Name, DataType = "nvarchar", Length = -1};
return new[] {column};
}
public static TableInfo GetRowFormat(List<XmlReadInstructions> instructions)
{
var res = new TableInfo(null);
foreach (var instruction in instructions)
{
foreach (var col in instruction.Columns)
{
if (res.Columns.Any(x => x.Name == col.Name)) continue;
res.Columns.Add(new ColumnInfo(res) { CommonType = new DbTypeString(), DataType = "nvarchar", Length = -1, Name = col.Name });
}
}
return res;
}
public TableObjectInfo(TableInfo table)
: base(table == null ? null : table.OwnerDatabase)
{
OwnerTable = table;
}
public override void AfterLoadLink()
{
base.AfterLoadLink();
if (RelatedTable == null)
{
RelatedTable = OwnerDatabase.GetTable(new NameWithSchema(_tableOrViewSchema, _tableOrViewName));
}
if (RelatedView == null)
{
RelatedView = OwnerDatabase.GetView(new NameWithSchema(_tableOrViewSchema, _tableOrViewName));
}
_tableOrViewName = null;
_tableOrViewSchema = null;
}