/// <summary>
/// Constructs MetaTable from the source data
/// </summary>
/// <param name="sourceTable">The source table.</param>
/// <param name="task">The task.</param>
/// <param name="logger">The logger.</param>
/// <returns></returns>
public MetaTable Construct(SourceTable sourceTable, TableExtractionTask task, ITextRender logger)
{
MetaTable metaTable = null;
switch (task.score.executionMode)
{
case ExtractionTaskEngineMode.Application:
metaTable = new MetaTable(task.tableDescription);
metaTable.SetSchema(sourceTable);
metaTable.ApplySchema(task.PropertyDictionary.items.Select(x => x.Meta));
metaTable.SetEntries(sourceTable);
metaTable.Comment = "Constructed by " + GetType().Name;
break;
case ExtractionTaskEngineMode.Validation:
metaTable = new MetaTable(task.tableDescription);
metaTable.SetSchema(sourceTable);
metaTable.SetEntriesAndLinkToSource(sourceTable); //.SetEntries(sourceTable);
metaTable.RefineSchema(sourceContentAnalysis);
metaTable.Comment = "Constructed by " + GetType().Name;
break;
default:
break;
}
return(metaTable);
}
public static SourceTable GetDistinctRows(this SourceTable source)
{
var all = source.GetContentCells(false);
all = TakeDistinct(all);
if (!all.Any())
{
return(null);
}
Int32 Width = all.Max(x => x.Count);
Int32 Height = all.Count;
var output = new SourceTable(Width, Height);
for (int i = 0; i < output.Height; i++)
{
var row = all[i];
for (int j = 0; j < output.Width; j++)
{
if (j < row.Count())
{
output.SetCell(j, i, row[j]);
// output[j, i].Value = row[j];
}
}
}
return(output);
}
public IQueryFactoryResult Process(IQueryContainer container)
{
var joinBuilder = new StringBuilder();
joinBuilder.Append($"{_joinAs.JoinType} JOIN {TargetTable.GetAlias().EnsureAlias()} AS {Alias.GetAlias().EnsureAlias()}" +
$" ON {Alias.GetAlias().EnsureAlias()}.{TargetColumn.ColumnName.EnsureAlias()}" +
$" = " +
$"{SourceTable.GetAlias().EnsureAlias()}.{SourceColumn.ColumnName.EnsureAlias()}");
var parameters = new List <IQueryParameter>();
if (Condition != null)
{
var condQueryBuilder = new ConditionalQueryBuilder();
condQueryBuilder.QueryBuilder.Append(" ");
foreach (var conditionCondition in Condition.Conditions)
{
conditionCondition.Render(condQueryBuilder, null);
}
joinBuilder.Append(condQueryBuilder.QueryBuilder);
parameters.AddRange(condQueryBuilder.QueryParameters);
}
return(new QueryFactoryResult(joinBuilder.ToString(), parameters.ToArray()));
}
/// <inheritdoc/>
public string ToDelimitedString()
{
System.Globalization.CultureInfo culture = System.Globalization.CultureInfo.CurrentCulture;
string separator = IsSubcomponent ? Configuration.SubcomponentSeparator : Configuration.ComponentSeparator;
return(string.Format(
culture,
StringHelper.StringFormatSequence(0, 25, separator),
PersonIdentifier,
FamilyName?.ToDelimitedString(),
GivenName,
SecondAndFurtherGivenNamesOrInitialsThereof,
Suffix,
Prefix,
Degree,
SourceTable?.ToDelimitedString(),
AssigningAuthority?.ToDelimitedString(),
NameTypeCode,
IdentifierCheckDigit,
CheckDigitScheme,
IdentifierTypeCode,
AssigningFacility?.ToDelimitedString(),
NameRepresentationCode,
NameContext?.ToDelimitedString(),
NameValidityRange?.ToDelimitedString(),
NameAssemblyOrder,
EffectiveDate.HasValue ? EffectiveDate.Value.ToString(Consts.DateTimeFormatPrecisionSecond, culture) : null,
ExpirationDate.HasValue ? ExpirationDate.Value.ToString(Consts.DateTimeFormatPrecisionSecond, culture) : null,
ProfessionalSuffix,
AssigningJurisdiction?.ToDelimitedString(),
AssigningAgencyOrDepartment?.ToDelimitedString(),
SecurityCheck,
SecurityCheckScheme
).TrimEnd(separator.ToCharArray()));
}
public virtual MetaTable Construct(SourceTable sourceTable, TableExtractionTask task)
{
var tb = universalMetaTableConstructor.Construct(sourceTable, task, output);
return(tb);
}
public void TableBaseMigration(SourceTable tbl, Converter[] converter, Join[] additional_joins)
{
// ------------------- Argument checks -------------------
if (tbl == null)
{
throw new ArgumentNullException("tbl");
}
if (tbl.DestinationObjectClass == null)
{
Log.InfoFormat("Skipping base migration of unmapped table [{0}]", tbl.Name);
return;
}
using (Log.InfoTraceMethodCallFormat("TableBaseMigration", "{0} to {1}", tbl.Name, tbl.DestinationObjectClass.Name))
{
// ------------------- Build columns -------------------
var mappedColumns = tbl.SourceColumn
.Where(c => c.DestinationProperty.Count > 0)
.OrderBy(c => c.Name)
.ToList();
// Ref Cols
var referringCols = mappedColumns.Where(c => c.References != null).ToList();
// ------------------- Migrate -------------------
if (referringCols.Count == 0 && (additional_joins == null || additional_joins.Length == 0))
{
TableBaseSimpleMigration(tbl, converter, mappedColumns);
}
else
{
TableBaseComplexMigration(tbl, converter, mappedColumns, referringCols, additional_joins);
}
}
}
/// <summary>
/// Gets the description.
/// </summary>
/// <param name="table">The table.</param>
/// <returns></returns>
public SourceTableDescription GetDescription(SourceTable table)
{
ContentStatMatrix contentStats = new ContentStatMatrix(table.Height, table.Width);
for (int i = 0; i < table.Height; i++)
{
for (int j = 0; j < table.Width; j++)
{
var t = DetermineContentType(table[j, i].Value);
contentStats.Assign(t, j, i);
}
}
contentStats.Compute();
SourceTableDescription output = new SourceTableDescription();
output.firstRowTest = MakeSliceTest(table, SourceTableSliceType.row);
output.firstColumnTest = MakeSliceTest(table, SourceTableSliceType.column);
output.valueZone = contentStats.GetValueZone();
output.matrix = contentStats;
output.sourceSize = new Data.primitives.coordinateXY()
{
x = table.Width,
y = table.Height
};
return(output);
}
private void CheckCircleReferences(Table table, TableMappingDefinition mappingDefinition)
{
if (_tablesToUpdateCircleReferences.ContainsKey(table.Name))
{
_tablesToUpdateCircleReferences[table.Name].ForEach(r =>
{
TableMappingDefinition temp = _tableMappingDefinitions.Single(d => d.DestinationTable.Name.Equals(r.OriginTableName));
FieldMappingDefinition fieldMappingDefinition = temp.FieldMappingDefinitions.SingleOrDefault(f => f.DestinationField.Name.Equals(r.OriginFieldName));
if (fieldMappingDefinition != null)
{
SourceTable sourceTable = temp.SourceTable;
Field sourceField = fieldMappingDefinition.SourceField;
DestinationTable destinationTable = temp.DestinationTable;
Field destinationField = fieldMappingDefinition.DestinationField;
FieldMappingInfo info = new FieldMappingInfo
{
DestinationField = destinationField,
DestinationTable = destinationTable,
SourceField = sourceField,
SourceTable = sourceTable
};
mappingDefinition.CircleReferences.Add(info);
}
});
}
}
public ETLInfo(ETLSource source, string dstTableName)
: this()
{
this.SourceTable = source;
this.DestinationTableName = dstTableName;
this.ColumnMapping = SourceTable.GetDeafultColumnMapping();
}
public static SourceColumn Column(this SourceTable tbl, string name)
{
if (tbl == null)
{
throw new ArgumentNullException("tbl");
}
return(tbl.SourceColumn.Single(i => i.Name == name));
}
private string DoConversion()
{
Logger.Info("Doing conversion");
List <string> scriptNames = new List <string>();
Dictionary <string, string> bcpExportCommands = new Dictionary <string, string>();
Dictionary <string, string> bcpImportCommands = new Dictionary <string, string>();
_destinationDatabase.Tables.ForEach(t =>
{
DestinationTable destinationTable = t;
SourceTable sourceTable = null;
try
{
Logger.Info("Processing " + destinationTable.Name);
TableMappingConfiguration mappingConfig = GetTableMappingConfig(destinationTable.Name);
sourceTable = GetSourceTable(destinationTable.Name, mappingConfig);
var mappingDefinition = CreateTableMappingDefinition(sourceTable, destinationTable, mappingConfig);
SqlGenerator sqlGenerator = new SqlGenerator(mappingDefinition, _options.CheckExistData);
BcpGenerator bcpGenerator = new BcpGenerator(_options.ServerName, _options.InstanceName, mappingDefinition, _options.BcpMode);
HandleBcp(sqlGenerator, bcpGenerator, mappingDefinition, bcpExportCommands, bcpImportCommands);
HandleMaxTextUpdate(sqlGenerator, mappingDefinition, scriptNames);
if (_options.BlobHandling)
{
HandleBlobUpdate(sqlGenerator, mappingDefinition, scriptNames);
}
}
catch (AppException ex)
{
if (ex.ErrorCode == AppExceptionCodes.DATABASE_ERROR_TABLE_NOT_FOUND)
{
Logger.Warn(destinationTable.Name + " not mapped");
}
}
});
// Generate bat file
string exportScript = BatGenerator.GenerateBcpExecuteBat(bcpExportCommands);
SaveToFile(_packageOutputPath, "BCP_Export.bat", exportScript);
Logger.Info("BCP Exporting data");
Process.Start(new ProcessStartInfo {
WorkingDirectory = _packageOutputPath, FileName = "BCP_Export.bat"
});
string importScript = BatGenerator.GenerateBcpExecuteBat(bcpImportCommands);
//SaveToFile(_packageOutputPath, "BCP_Import.bat", importScript);
List <string> scriptPaths = scriptNames.Select(s => Path.Combine(CONVERSION_PACKAGE_FOLDER, s)).ToList();
string updateScript = BatGenerator.GenerateSqlExecuteBat(scriptPaths, _options.ServerName, _options.InstanceName, _options.Username, _options.Password);
//SaveToFile(_packageOutputPath, "Update_Blob_Text.bat", updateScript);
string batScript = importScript + Environment.NewLine + updateScript;
return(batScript);
}
private void AddQueryButton_Click(object sender, EventArgs e)
{
// Find a name like "query1", "query2", etc. that doesn't exist in the notebook and isn't to be imported.
// First make a list of all the existing names that start with "query" so we can avoid them.
var notebook = _manager.Notebook;
List <string> existingNames = new();
using (var sdt = notebook.Query("SELECT name FROM sqlite_master")) {
foreach (var row in sdt.Rows)
{
var name = (string)row[0];
if (name.StartsWith("query", StringComparison.OrdinalIgnoreCase))
{
existingNames.Add((string)row[0]);
}
}
}
foreach (DataRow row in _dataTable.Rows)
{
var t = (SourceTable)row["source_table"];
existingNames.Add(t.TargetTableName);
}
// Count up until we find a unique name.
string targetName;
for (var i = 1; ; i++)
{
var proposedTargetName = $"query{i}";
if (!existingNames.Any(x => x.Equals(proposedTargetName, StringComparison.OrdinalIgnoreCase)))
{
targetName = proposedTargetName;
break;
}
}
// Ask the user for a query.
DatabaseImportCustomQueryForm f = new(_session, targetName, "SELECT * FROM ");
var result = f.ShowDialog(this);
Focus();
if (result != DialogResult.OK)
{
return;
}
// Add a SourceTable to the list for this query. Check it by default.
var sourceTable = SourceTable.FromSql(f.Sql, f.TargetName);
var newRow = _dataTable.NewRow();
newRow["source_table"] = sourceTable;
newRow["display_name"] = sourceTable.DisplayText;
newRow["to_be_imported"] = true;
_dataTable.Rows.Add(newRow);
}
public static SourceTable Merge(this IEnumerable <List <List <String> > > contentSets, Boolean asColumns, Boolean OnlyDistinct)
{
var all = new List <List <String> >();
foreach (var cset in contentSets)
{
all.AddRange(cset);
}
if (OnlyDistinct)
{
all = TakeDistinct(all);
}
Int32 Width = all.Max(x => x.Count);
SourceTable output = null;
if (asColumns)
{
output = new SourceTable(all.Count, Width);
for (int i = 0; i < output.Width; i++)
{
var column = all[i];
for (int j = 0; j < output.Height; j++)
{
if (j < column.Count)
{
output[i, j].Value = column[j];
}
}
}
}
else
{
output = new SourceTable(Width, all.Count);
for (int i = 0; i < output.Height; i++)
{
var row = all[i];
for (int j = 0; j < output.Width; j++)
{
if (j < row.Count())
{
output[j, i].Value = row[j];
}
}
}
}
return(output);
}
public IQueryFactoryResult Process(IQueryContainer container)
{
var joinBuilder = new StringBuilder();
joinBuilder.Append($"{_joinAs.JoinType} JOIN [{TargetTable.GetAlias()}] AS [{Alias.GetAlias()}]" +
$" ON [{Alias.GetAlias()}].[{TargetColumn.ColumnName.TrimAlias()}]" +
$" = " +
$"[{SourceTable.GetAlias()}].[{SourceColumn.ColumnName}]");
return(new QueryFactoryResult(joinBuilder.ToString()));
}
/// <summary>
/// Gets or sets a value indicating whether [trim input].
/// </summary>
/// <value>
/// <c>true</c> if [trim input]; otherwise, <c>false</c>.
/// </value>
/*
* public List<String> NoDataWillCards
* {
* get
* {
* if (!_noDataWillCards.Any())
* {
* return DefaultNoDataWillcads;
* }
* return _noDataWillCards;
* }
* set
* {
* if (value.SequenceEqual(DefaultNoDataWillcads))
* {
* return;
* }
* _noDataWillCards = value;
* }
* }*/
/// <summary>
/// Performs a test on
/// </summary>
/// <param name="sourceTable">The source table.</param>
/// <param name="type">The type.</param>
/// <param name="i">The i.</param>
/// <returns></returns>
public SourceTableSliceTest MakeSliceTest(SourceTable sourceTable, SourceTableSliceType type, Int32 i = 0)
{
SourceTableSliceTest output = new SourceTableSliceTest()
{
format = type
};
switch (type)
{
case SourceTableSliceType.row:
output.Values = sourceTable.GetRow(i);
break;
default:
case SourceTableSliceType.column:
output.Values = sourceTable.GetColumn(i);
break;
}
// CellContentType contentType = CellContentType.unknown;
foreach (var v in output.Values)
{
output.ValueStats.Assign(DetermineContentType(v));
//if (contentType == CellContentType.unknown)
//{
// contentType = t.type;
//} else if (contentType != t.type)
//{
// if (!t.type.HasFlag(contentType))
// {
// output.IsUniformFormat = false;
// }
//}
if (v.IsNullOrEmpty())
{
output.IsNoEmptyValue = false;
}
if (!output.DistinctValues.Contains(v))
{
output.DistinctValues.Add(v);
}
}
output.ValueStats.Compute();
output.IsUniformFormat = output.ValueStats.IsUniformFormat();
if (output.DistinctValues.Count < output.Values.Count)
{
output.IsDistinctValue = false;
}
return(output);
}
public void CleanDestination(SourceTable tbl)
{
if (tbl == null)
throw new ArgumentNullException("tbl");
if (tbl.DestinationObjectClass == null)
{
Log.InfoFormat("Skipping cleaning of unmapped table [{0}]", tbl.Name);
return;
}
CleanDestination(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName));
}
/// <summary>
/// Performs a test on
/// </summary>
/// <param name="sourceTable">The source table.</param>
/// <param name="type">The type.</param>
/// <param name="i">The i.</param>
/// <returns></returns>
public SourceTableSliceTest MakeSliceTest(SourceTable sourceTable, SourceTableSliceType type, Int32 i = 0)
{
SourceTableSliceTest output = new SourceTableSliceTest()
{
format = type
};
switch (type)
{
case SourceTableSliceType.row:
output.Values = sourceTable.GetRow(i);
break;
default:
case SourceTableSliceType.column:
output.Values = sourceTable.GetColumn(i);
break;
}
CellContentType contentType = CellContentType.unknown;
List <String> DistinctValues = new List <string>();
foreach (var v in output.Values)
{
CellContentInfo t = sourceContentAnalysis.DetermineContentType(v);
if (contentType == CellContentType.unknown)
{
contentType = t.type;
}
else if (contentType != t.type)
{
if (!t.type.HasFlag(contentType))
{
output.IsUniformFormat = false;
}
}
if (v.IsNullOrEmpty())
{
output.IsNoEmptyValue = false;
}
if (!DistinctValues.Contains(v))
{
DistinctValues.Add(v);
}
}
if (DistinctValues.Count < output.Values.Count)
{
output.IsDistinctValue = false;
}
return(output);
}
public void CleanDestination(SourceTable tbl)
{
if (tbl == null)
{
throw new ArgumentNullException("tbl");
}
if (tbl.DestinationObjectClass == null)
{
Log.InfoFormat("Skipping cleaning of unmapped table [{0}]", tbl.Name);
return;
}
CleanDestination(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName));
}
public bool Add(SourceTable model)
{
bool result = false;
_repository.Add(model, (item, err) =>
{
if (err == null)
{
result = true;
}
});
return(result);
}
public bool Update(SourceTable model, int tkey)
{
bool _result = false;
_repository.Update(model, (item, err) =>
{
if (err == null)
{
_result = true;
}
});
return(_result);
}
public void BulkOfEntitiesConsistent()
{
using (var dl = new DisposableList())
{
// create the source table
var sourceTableWrapper = new TableWrapper(TestHelpers.GenUnique(TestConfig.TableName), TestConfig.PrimaryStorageAccount, true);
dl.Add(sourceTableWrapper.Delete);
// initialize the target table and attach it to the disposable container
var targetTable = new TableWrapper(TestHelpers.GenUnique(TestConfig.TableName), TestConfig.SecondaryStorageAccount, true);
dl.Add(targetTable.Delete);
var tableParamsResolver = new InMemoryTargetTableResolver();
tableParamsResolver.Add(TestHelpers.TableKey, targetTable);
// create gateway blob storage
var gateBlob = new InMemoryGatewayBlobStore();
// create a gateway queue
var gateQueue = new QueueWrapper(TestHelpers.GenUnique("gateq"), TestConfig.GatewayStorageAccount);
var gateQueueWriter = new GatewayQueueWriter(gateQueue, gateBlob);
var sourceTable = new SourceTable <StarEntity>(sourceTableWrapper, gateQueueWriter, TestHelpers.TableKey);
// 100 entities to satisfy TableStorage's batch requirements
var entities = TestHelpers.CreateStarEntities(100);
sourceTable.Write(entities);
// Now verify that the entities were synced to the secondary table storage
TableGatewayQueueProcessor gateQueueProcessor = new TableGatewayQueueProcessor(
new GatewayQueueReader(gateQueue, gateBlob),
tableParamsResolver
);
bool result = gateQueueProcessor.ProcessNext();
Assert.IsTrue(result);
var targetEntities = targetTable.ReadEntities <StarEntity>()
.ToList();
Assert.AreEqual(entities.Count, targetEntities.Count);
foreach (var entity in entities)
{
TestHelpers.AssertEqualStars(
entity,
targetEntities.First(x => x.RowKey == entity.RowKey)
);
}
}
}
public Translator(SourceTable tbl, IDataReader source, IEnumerable <SourceColumn> srcColumns, Converter[] converter)
{
if (tbl == null)
{
throw new ArgumentNullException("tbl");
}
if (source == null)
{
throw new ArgumentNullException("source");
}
if (srcColumns == null)
{
throw new ArgumentNullException("srcColumns");
}
//_tbl = tbl;
_source = source;
_srcColumns = srcColumns.ToArray();
_srcColumnsInfos = _srcColumns.Select(
col => new SourceColumnInfo(DbTypeMapper.GetDbTypeForProperty(col.DestinationProperty.Last().GetType()),
DbTypeMapper.GetDbType(col.DbType),
col.DestinationProperty.Last() is EnumerationProperty,
col.DestinationProperty.First() is CompoundObjectProperty)).ToArray();
_converter = converter != null?converter.ToDictionary(c => c.Column) : new Dictionary <SourceColumn, Converter>();
_resultColumnCount = _srcColumns.Length;
if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType()))
{
// TODO: That's a bad hack!
_errorColIdx = _resultColumnCount;
_resultColumnCount++;
}
else
{
_errorColIdx = -1;
}
foreach (var comp in srcColumns
.Where(c => c.DestinationProperty.First() is CompoundObjectProperty)
.GroupBy(c => c.DestinationProperty.First()))
{
foreach (var col in comp)
{
_compoundObjectSourceColumns[col.Name] = _resultColumnCount;
}
_resultColumnCount++;
}
}
public MetaTableDescription ConstructDescription(SourceTable sourceTable, TableExtractionTask task, ITextRender logger)
{
//if (sourceTable[0, 0].Value.isNullOrEmpty())
//{
// sourceTable[0, 0].Value = "ID";
//}
MetaTableDescription metaDescription = null;
builderForText reporter = task.score.CurrentEntry().reporter;
switch (task.score.executionMode)
{
case ExtractionTaskEngineMode.Training:
SourceTableDescription sourceDesc = sourceContentAnalysis.GetDescription(sourceTable);
metaDescription = new imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableDescription(sourceDesc, imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableFormatType.vertical);
// task.tableDescription = metaDescription;
//task.score.CurrentEntry().metaTableDescription = metaDescription;
break;
case ExtractionTaskEngineMode.Validation:
default:
case ExtractionTaskEngineMode.Application:
if (task.tableDescription == null)
{
throw new Exception("Task [" + task.name + "] has no table description set.");
}
break;
}
if (folder != null)
{
String sp = folder.pathFor("UMTC_Construct_" + task.name + "_" + task.score.executionMode.toString() + ".txt");
File.WriteAllText(sp, reporter.GetContent());
}
return(metaDescription);
}
public void SetEntriesAndLinkToSource(SourceTable source)
{
Int32 Skip = description.EntrySkipCount;
List <List <SourceTableCell> > data = source.GetContentCells(description.format == MetaTableFormatType.vertical);
sourceTable = source;
for (int i = Skip; i < data.Count; i++)
{
if (i != description.index_propertyID)
{
entries.CreateEntry(data[i], true);
}
}
}
private TableMappingDefinition CreateTableMappingDefinition(SourceTable srcTable, DestinationTable destTable, TableMappingConfiguration mappingConfig)
{
TableMappingDefinition mappingDefinition;
if (mappingConfig != null)
{
mappingDefinition = new TableMappingDefinition(srcTable, destTable, mappingConfig.FieldMappings);
}
else
{
mappingDefinition = new TableMappingDefinition(srcTable, destTable);
}
return(mappingDefinition);
}
private static List <string> GetDestinationColumnNames(SourceTable tbl, List <SourceColumn> srcColumns)
{
var dstColumnNames = srcColumns.Select(c => GetColName(c.DestinationProperty)).ToList();
// Error Col
if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType()))
{
dstColumnNames.Add("MigrationErrors");
}
dstColumnNames.AddRange(srcColumns
.Where(c => c.DestinationProperty.First() is CompoundObjectProperty)
.GroupBy(c => c.DestinationProperty.First())
.Select(grp => GetColName(grp.Key)));
return(dstColumnNames);
}
private static DataTable GetTables(IImportSession session)
{
DataTable dt = new();
dt.Columns.Add("source_table", typeof(SourceTable));
dt.Columns.Add("display_name", typeof(string));
dt.Columns.Add("to_be_imported", typeof(bool));
dt.BeginLoadData();
foreach (var(schema, table) in session.TableNames)
{
var sourceTable = SourceTable.FromTable(schema, table);
dt.LoadDataRow(new object[] { sourceTable, sourceTable.DisplayText, false }, true);
}
dt.EndLoadData();
return(dt);
}
public override SourceTable MakeSourceTable(HtmlNode divNode)
{
var head = divNode.SelectSingleNode(divNode.XPath.add("div[not(*)]", "//"));
if (head == null)
{
head = divNode;
}
List <HtmlNode> html_tablerows = AdaptiveRowSelection(head, AdaptiveSteps); //divNode.SelectNodesInDepthRange(x => x.Name.Equals(TableSelectionTag, StringComparison.InvariantCultureIgnoreCase), TableSelectionDepthLimit, TableSelectionDepthStart, false);
List <TableHtmlSourceRow> html_selected_rows = new List <TableHtmlSourceRow>();
Int32 columns = 0;
foreach (HtmlNode row in html_tablerows)
{
TableHtmlSourceRow sourceRow = SetSourceRow(row);
columns = Math.Max(columns, sourceRow.Count);
if (sourceRow.Count > 0)
{
html_selected_rows.Add(sourceRow);
}
}
Int32 rows = html_selected_rows.Count;
SourceTable sourceTable = new SourceTable(columns, rows);
if (!sourceTable.IsValid)
{
if (DoThrowException)
{
throw new ArgumentOutOfRangeException(nameof(columns), "Number of columns and rows can't be less than 1. Specified values: width [" + columns.ToString() + "], height [" + rows.ToString() + "]");
}
}
for (int i = 0; i < html_selected_rows.Count; i++)
{
TableHtmlSourceRow row_node = html_selected_rows[i];
for (int j = 0; j < row_node.RowCells.Count; j++)
{
SetSourceTableCell(sourceTable[j, i], row_node.RowCells[j], divNode.OwnerDocument);
}
}
return(sourceTable);
}
private void TableBaseSimpleMigration(SourceTable tbl, Converter[] nullConverter, List <SourceColumn> mappedColumns)
{
var dstColumnNames = GetDestinationColumnNames(tbl, mappedColumns);
var srcColumnNames = mappedColumns.Select(c => c.Name).ToArray();
var tblRef = _src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name);
// no fk mapping required
using (var srcReader = _src.ReadTableData(tblRef, srcColumnNames))
using (var translator = new Translator(tbl, srcReader, mappedColumns, nullConverter))
{
_dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames);
WriteLog(
tbl.Name, translator.ProcessedRows,
tbl.DestinationObjectClass.TableName, translator.ProcessedRows);
}
}
private SourceTable GetSourceTable(string destTableName, TableMappingConfiguration mappingConfig)
{
SourceTable sourceTable = null;
if (mappingConfig != null)
{
string mapSourceTableName = mappingConfig.SourceTableName != null ? mappingConfig.SourceTableName : destTableName;
sourceTable = _sourceDatabase.GetTable(mapSourceTableName);
}
else
{
sourceTable = _sourceDatabase.GetTable(destTableName);
}
return(sourceTable);
}
private void HandleMaxTextUpdate(SqlGenerator sqlGenerator, TableMappingDefinition mappingDefinition, List <string> scriptNames)
{
Logger.Info("Handling max text");
SourceTable srcTable = mappingDefinition.SourceTable;
DestinationTable destTable = mappingDefinition.DestinationTable;
string postSqlPackagePath = Path.Combine(_packageOutputPath, CONVERSION_PACKAGE_FOLDER);
Logger.Info("Generating script for updating max text fields");
string script = sqlGenerator.GenerateVarCharMaxUpdateScript();
if (!string.IsNullOrEmpty(script))
{
var fileName = string.Format("{0}-{1}_TEXT.sql", srcTable.Name, destTable.Name);
SaveToFile(postSqlPackagePath, fileName, script);
scriptNames.Add(fileName);
}
}
public Translator(SourceTable tbl, IDataReader source, IEnumerable<SourceColumn> srcColumns, Converter[] converter)
{
if (tbl == null) throw new ArgumentNullException("tbl");
if (source == null) throw new ArgumentNullException("source");
if (srcColumns == null) throw new ArgumentNullException("srcColumns");
//_tbl = tbl;
_source = source;
_srcColumns = srcColumns.ToArray();
_srcColumnsInfos = _srcColumns.Select(
col => new SourceColumnInfo(DbTypeMapper.GetDbTypeForProperty(col.DestinationProperty.Last().GetType()),
DbTypeMapper.GetDbType(col.DbType),
col.DestinationProperty.Last() is EnumerationProperty,
col.DestinationProperty.First() is CompoundObjectProperty)).ToArray();
_converter = converter != null ? converter.ToDictionary(c => c.Column) : new Dictionary<SourceColumn, Converter>();
_resultColumnCount = _srcColumns.Length;
if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType()))
{
// TODO: That's a bad hack!
_errorColIdx = _resultColumnCount;
_resultColumnCount++;
}
else
{
_errorColIdx = -1;
}
foreach (var comp in srcColumns
.Where(c => c.DestinationProperty.First() is CompoundObjectProperty)
.GroupBy(c => c.DestinationProperty.First()))
{
foreach (var col in comp)
{
_compoundObjectSourceColumns[col.Name] = _resultColumnCount;
}
_resultColumnCount++;
}
}
public void TableBaseMigration(SourceTable tbl, Converter[] converter, Join[] additional_joins)
{
// ------------------- Argument checks -------------------
if (tbl == null)
throw new ArgumentNullException("tbl");
if (tbl.DestinationObjectClass == null)
{
Log.InfoFormat("Skipping base migration of unmapped table [{0}]", tbl.Name);
return;
}
using (Log.InfoTraceMethodCallFormat("TableBaseMigration", "{0} to {1}", tbl.Name, tbl.DestinationObjectClass.Name))
{
// ------------------- Build columns -------------------
var mappedColumns = tbl.SourceColumn
.Where(c => c.DestinationProperty.Count > 0)
.OrderBy(c => c.Name)
.ToList();
// Ref Cols
var referringCols = mappedColumns.Where(c => c.References != null).ToList();
// ------------------- Migrate -------------------
if (referringCols.Count == 0 && (additional_joins == null || additional_joins.Length == 0))
{
TableBaseSimpleMigration(tbl, converter, mappedColumns);
}
else
{
TableBaseComplexMigration(tbl, converter, mappedColumns, referringCols, additional_joins);
}
}
}
private static List<string> GetDestinationColumnNames(SourceTable tbl, List<SourceColumn> srcColumns)
{
var dstColumnNames = srcColumns.Select(c => GetColName(c.DestinationProperty)).ToList();
// Error Col
if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType()))
{
dstColumnNames.Add("MigrationErrors");
}
dstColumnNames.AddRange(srcColumns
.Where(c => c.DestinationProperty.First() is CompoundObjectProperty)
.GroupBy(c => c.DestinationProperty.First())
.Select(grp => GetColName(grp.Key)));
return dstColumnNames;
}
private void TableBaseComplexMigration(SourceTable tbl, Converter[] converter, List<SourceColumn> mappedColumns, List<SourceColumn> referringCols, Join[] additional_joins)
{
if (additional_joins != null
&& additional_joins.Length > 0
&& additional_joins.All(j => referringCols
.Select(c => c.DestinationProperty.Single())
.OfType<ObjectReferenceProperty>()
.Any(orp => j.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName))))
{
throw new InvalidOperationException("Unmapped additional joins found");
}
// could automatically create needed indices
var all_joins = new Dictionary<SourceColumn, Join>();
var root_joins = referringCols
.GroupBy(k => k.DestinationProperty.Single())
.SelectMany(referenceGroup => CreateReferenceJoin(referenceGroup, all_joins))
.ToArray();
// Add manual joins
IEnumerable<Join> joins;
if (additional_joins != null)
{
joins = root_joins.Union(additional_joins);
}
else
{
joins = root_joins;
}
var srcColumns = mappedColumns
.Where(c => c.References == null)
.Union(
referringCols
.GroupBy(k => k.DestinationProperty.Single()) // referring columns cannot be mapped remotely
.Select(g => g.First(p => p.References.References == null))
).ToList();
var srcColumnNames = srcColumns.Select(c =>
{
var orp = c.DestinationProperty.FirstOrDefault() as ObjectReferenceProperty;
if (c.References != null)
{
return new ProjectionColumn("ID", all_joins[c], System.Data.DbType.Int32, c.DestinationProperty.Single().Name);
}
else if (c.References == null
&& orp != null)
{
if (additional_joins != null
&& additional_joins.Count(i => i.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName)) > 0)
{
return new ProjectionColumn(
"ID",
additional_joins.Single(j => j.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName)),
System.Data.DbType.Int32,
orp.Name);
}
else if (converter.Any(cnv => cnv.Column.Name == c.Name))
{
return new ProjectionColumn(c.Name, ColumnRef.PrimaryTable, c.DestinationProperty.Single().Name);
}
else
{
throw new InvalidOperationException(string.Format("No join found for {0}", c));
}
}
else
{
return new ProjectionColumn(c.Name, ColumnRef.PrimaryTable, (System.Data.DbType)c.DbType, null);
}
}).ToList();
var dstColumnNames = GetDestinationColumnNames(tbl, srcColumns);
long processedRows;
using (var srcReader = _src.ReadJoin(_src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name), srcColumnNames, joins))
using (var translator = new Translator(tbl, srcReader, srcColumns, converter))
{
_dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames);
processedRows = translator.ProcessedRows;
}
// count rows in original table, joins should not add or remove rows
WriteLog(
tbl.Name, _src.CountRows(_src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name)),
tbl.DestinationObjectClass.TableName, processedRows);
}
private void TableBaseSimpleMigration(SourceTable tbl, Converter[] nullConverter, List<SourceColumn> mappedColumns)
{
var dstColumnNames = GetDestinationColumnNames(tbl, mappedColumns);
var srcColumnNames = mappedColumns.Select(c => c.Name).ToArray();
var tblRef = _src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name);
// no fk mapping required
using (var srcReader = _src.ReadTableData(tblRef, srcColumnNames))
using (var translator = new Translator(tbl, srcReader, mappedColumns, nullConverter))
{
_dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames);
WriteLog(
tbl.Name, translator.ProcessedRows,
tbl.DestinationObjectClass.TableName, translator.ProcessedRows);
}
}
public void TableBaseMigration(SourceTable tbl)
{
TableBaseMigration(tbl, (Converter[])null, (Join[])null);
}
public void TableBaseMigration(SourceTable tbl, params Converter[] converter)
{
TableBaseMigration(tbl, converter, null);
}
