/// <summary> /// Constructs MetaTable from the source data /// </summary> /// <param name="sourceTable">The source table.</param> /// <param name="task">The task.</param> /// <param name="logger">The logger.</param> /// <returns></returns> public MetaTable Construct(SourceTable sourceTable, TableExtractionTask task, ITextRender logger) { MetaTable metaTable = null; switch (task.score.executionMode) { case ExtractionTaskEngineMode.Application: metaTable = new MetaTable(task.tableDescription); metaTable.SetSchema(sourceTable); metaTable.ApplySchema(task.PropertyDictionary.items.Select(x => x.Meta)); metaTable.SetEntries(sourceTable); metaTable.Comment = "Constructed by " + GetType().Name; break; case ExtractionTaskEngineMode.Validation: metaTable = new MetaTable(task.tableDescription); metaTable.SetSchema(sourceTable); metaTable.SetEntriesAndLinkToSource(sourceTable); //.SetEntries(sourceTable); metaTable.RefineSchema(sourceContentAnalysis); metaTable.Comment = "Constructed by " + GetType().Name; break; default: break; } return(metaTable); }
public static SourceTable GetDistinctRows(this SourceTable source) { var all = source.GetContentCells(false); all = TakeDistinct(all); if (!all.Any()) { return(null); } Int32 Width = all.Max(x => x.Count); Int32 Height = all.Count; var output = new SourceTable(Width, Height); for (int i = 0; i < output.Height; i++) { var row = all[i]; for (int j = 0; j < output.Width; j++) { if (j < row.Count()) { output.SetCell(j, i, row[j]); // output[j, i].Value = row[j]; } } } return(output); }
public IQueryFactoryResult Process(IQueryContainer container) { var joinBuilder = new StringBuilder(); joinBuilder.Append($"{_joinAs.JoinType} JOIN {TargetTable.GetAlias().EnsureAlias()} AS {Alias.GetAlias().EnsureAlias()}" + $" ON {Alias.GetAlias().EnsureAlias()}.{TargetColumn.ColumnName.EnsureAlias()}" + $" = " + $"{SourceTable.GetAlias().EnsureAlias()}.{SourceColumn.ColumnName.EnsureAlias()}"); var parameters = new List <IQueryParameter>(); if (Condition != null) { var condQueryBuilder = new ConditionalQueryBuilder(); condQueryBuilder.QueryBuilder.Append(" "); foreach (var conditionCondition in Condition.Conditions) { conditionCondition.Render(condQueryBuilder, null); } joinBuilder.Append(condQueryBuilder.QueryBuilder); parameters.AddRange(condQueryBuilder.QueryParameters); } return(new QueryFactoryResult(joinBuilder.ToString(), parameters.ToArray())); }
/// <inheritdoc/> public string ToDelimitedString() { System.Globalization.CultureInfo culture = System.Globalization.CultureInfo.CurrentCulture; string separator = IsSubcomponent ? Configuration.SubcomponentSeparator : Configuration.ComponentSeparator; return(string.Format( culture, StringHelper.StringFormatSequence(0, 25, separator), PersonIdentifier, FamilyName?.ToDelimitedString(), GivenName, SecondAndFurtherGivenNamesOrInitialsThereof, Suffix, Prefix, Degree, SourceTable?.ToDelimitedString(), AssigningAuthority?.ToDelimitedString(), NameTypeCode, IdentifierCheckDigit, CheckDigitScheme, IdentifierTypeCode, AssigningFacility?.ToDelimitedString(), NameRepresentationCode, NameContext?.ToDelimitedString(), NameValidityRange?.ToDelimitedString(), NameAssemblyOrder, EffectiveDate.HasValue ? EffectiveDate.Value.ToString(Consts.DateTimeFormatPrecisionSecond, culture) : null, ExpirationDate.HasValue ? ExpirationDate.Value.ToString(Consts.DateTimeFormatPrecisionSecond, culture) : null, ProfessionalSuffix, AssigningJurisdiction?.ToDelimitedString(), AssigningAgencyOrDepartment?.ToDelimitedString(), SecurityCheck, SecurityCheckScheme ).TrimEnd(separator.ToCharArray())); }
public virtual MetaTable Construct(SourceTable sourceTable, TableExtractionTask task) { var tb = universalMetaTableConstructor.Construct(sourceTable, task, output); return(tb); }
public void TableBaseMigration(SourceTable tbl, Converter[] converter, Join[] additional_joins) { // ------------------- Argument checks ------------------- if (tbl == null) { throw new ArgumentNullException("tbl"); } if (tbl.DestinationObjectClass == null) { Log.InfoFormat("Skipping base migration of unmapped table [{0}]", tbl.Name); return; } using (Log.InfoTraceMethodCallFormat("TableBaseMigration", "{0} to {1}", tbl.Name, tbl.DestinationObjectClass.Name)) { // ------------------- Build columns ------------------- var mappedColumns = tbl.SourceColumn .Where(c => c.DestinationProperty.Count > 0) .OrderBy(c => c.Name) .ToList(); // Ref Cols var referringCols = mappedColumns.Where(c => c.References != null).ToList(); // ------------------- Migrate ------------------- if (referringCols.Count == 0 && (additional_joins == null || additional_joins.Length == 0)) { TableBaseSimpleMigration(tbl, converter, mappedColumns); } else { TableBaseComplexMigration(tbl, converter, mappedColumns, referringCols, additional_joins); } } }
/// <summary> /// Gets the description. /// </summary> /// <param name="table">The table.</param> /// <returns></returns> public SourceTableDescription GetDescription(SourceTable table) { ContentStatMatrix contentStats = new ContentStatMatrix(table.Height, table.Width); for (int i = 0; i < table.Height; i++) { for (int j = 0; j < table.Width; j++) { var t = DetermineContentType(table[j, i].Value); contentStats.Assign(t, j, i); } } contentStats.Compute(); SourceTableDescription output = new SourceTableDescription(); output.firstRowTest = MakeSliceTest(table, SourceTableSliceType.row); output.firstColumnTest = MakeSliceTest(table, SourceTableSliceType.column); output.valueZone = contentStats.GetValueZone(); output.matrix = contentStats; output.sourceSize = new Data.primitives.coordinateXY() { x = table.Width, y = table.Height }; return(output); }
private void CheckCircleReferences(Table table, TableMappingDefinition mappingDefinition) { if (_tablesToUpdateCircleReferences.ContainsKey(table.Name)) { _tablesToUpdateCircleReferences[table.Name].ForEach(r => { TableMappingDefinition temp = _tableMappingDefinitions.Single(d => d.DestinationTable.Name.Equals(r.OriginTableName)); FieldMappingDefinition fieldMappingDefinition = temp.FieldMappingDefinitions.SingleOrDefault(f => f.DestinationField.Name.Equals(r.OriginFieldName)); if (fieldMappingDefinition != null) { SourceTable sourceTable = temp.SourceTable; Field sourceField = fieldMappingDefinition.SourceField; DestinationTable destinationTable = temp.DestinationTable; Field destinationField = fieldMappingDefinition.DestinationField; FieldMappingInfo info = new FieldMappingInfo { DestinationField = destinationField, DestinationTable = destinationTable, SourceField = sourceField, SourceTable = sourceTable }; mappingDefinition.CircleReferences.Add(info); } }); } }
public ETLInfo(ETLSource source, string dstTableName) : this() { this.SourceTable = source; this.DestinationTableName = dstTableName; this.ColumnMapping = SourceTable.GetDeafultColumnMapping(); }
public static SourceColumn Column(this SourceTable tbl, string name) { if (tbl == null) { throw new ArgumentNullException("tbl"); } return(tbl.SourceColumn.Single(i => i.Name == name)); }
private string DoConversion() { Logger.Info("Doing conversion"); List <string> scriptNames = new List <string>(); Dictionary <string, string> bcpExportCommands = new Dictionary <string, string>(); Dictionary <string, string> bcpImportCommands = new Dictionary <string, string>(); _destinationDatabase.Tables.ForEach(t => { DestinationTable destinationTable = t; SourceTable sourceTable = null; try { Logger.Info("Processing " + destinationTable.Name); TableMappingConfiguration mappingConfig = GetTableMappingConfig(destinationTable.Name); sourceTable = GetSourceTable(destinationTable.Name, mappingConfig); var mappingDefinition = CreateTableMappingDefinition(sourceTable, destinationTable, mappingConfig); SqlGenerator sqlGenerator = new SqlGenerator(mappingDefinition, _options.CheckExistData); BcpGenerator bcpGenerator = new BcpGenerator(_options.ServerName, _options.InstanceName, mappingDefinition, _options.BcpMode); HandleBcp(sqlGenerator, bcpGenerator, mappingDefinition, bcpExportCommands, bcpImportCommands); HandleMaxTextUpdate(sqlGenerator, mappingDefinition, scriptNames); if (_options.BlobHandling) { HandleBlobUpdate(sqlGenerator, mappingDefinition, scriptNames); } } catch (AppException ex) { if (ex.ErrorCode == AppExceptionCodes.DATABASE_ERROR_TABLE_NOT_FOUND) { Logger.Warn(destinationTable.Name + " not mapped"); } } }); // Generate bat file string exportScript = BatGenerator.GenerateBcpExecuteBat(bcpExportCommands); SaveToFile(_packageOutputPath, "BCP_Export.bat", exportScript); Logger.Info("BCP Exporting data"); Process.Start(new ProcessStartInfo { WorkingDirectory = _packageOutputPath, FileName = "BCP_Export.bat" }); string importScript = BatGenerator.GenerateBcpExecuteBat(bcpImportCommands); //SaveToFile(_packageOutputPath, "BCP_Import.bat", importScript); List <string> scriptPaths = scriptNames.Select(s => Path.Combine(CONVERSION_PACKAGE_FOLDER, s)).ToList(); string updateScript = BatGenerator.GenerateSqlExecuteBat(scriptPaths, _options.ServerName, _options.InstanceName, _options.Username, _options.Password); //SaveToFile(_packageOutputPath, "Update_Blob_Text.bat", updateScript); string batScript = importScript + Environment.NewLine + updateScript; return(batScript); }
private void AddQueryButton_Click(object sender, EventArgs e) { // Find a name like "query1", "query2", etc. that doesn't exist in the notebook and isn't to be imported. // First make a list of all the existing names that start with "query" so we can avoid them. var notebook = _manager.Notebook; List <string> existingNames = new(); using (var sdt = notebook.Query("SELECT name FROM sqlite_master")) { foreach (var row in sdt.Rows) { var name = (string)row[0]; if (name.StartsWith("query", StringComparison.OrdinalIgnoreCase)) { existingNames.Add((string)row[0]); } } } foreach (DataRow row in _dataTable.Rows) { var t = (SourceTable)row["source_table"]; existingNames.Add(t.TargetTableName); } // Count up until we find a unique name. string targetName; for (var i = 1; ; i++) { var proposedTargetName = $"query{i}"; if (!existingNames.Any(x => x.Equals(proposedTargetName, StringComparison.OrdinalIgnoreCase))) { targetName = proposedTargetName; break; } } // Ask the user for a query. DatabaseImportCustomQueryForm f = new(_session, targetName, "SELECT * FROM "); var result = f.ShowDialog(this); Focus(); if (result != DialogResult.OK) { return; } // Add a SourceTable to the list for this query. Check it by default. var sourceTable = SourceTable.FromSql(f.Sql, f.TargetName); var newRow = _dataTable.NewRow(); newRow["source_table"] = sourceTable; newRow["display_name"] = sourceTable.DisplayText; newRow["to_be_imported"] = true; _dataTable.Rows.Add(newRow); }
public static SourceTable Merge(this IEnumerable <List <List <String> > > contentSets, Boolean asColumns, Boolean OnlyDistinct) { var all = new List <List <String> >(); foreach (var cset in contentSets) { all.AddRange(cset); } if (OnlyDistinct) { all = TakeDistinct(all); } Int32 Width = all.Max(x => x.Count); SourceTable output = null; if (asColumns) { output = new SourceTable(all.Count, Width); for (int i = 0; i < output.Width; i++) { var column = all[i]; for (int j = 0; j < output.Height; j++) { if (j < column.Count) { output[i, j].Value = column[j]; } } } } else { output = new SourceTable(Width, all.Count); for (int i = 0; i < output.Height; i++) { var row = all[i]; for (int j = 0; j < output.Width; j++) { if (j < row.Count()) { output[j, i].Value = row[j]; } } } } return(output); }
public IQueryFactoryResult Process(IQueryContainer container) { var joinBuilder = new StringBuilder(); joinBuilder.Append($"{_joinAs.JoinType} JOIN [{TargetTable.GetAlias()}] AS [{Alias.GetAlias()}]" + $" ON [{Alias.GetAlias()}].[{TargetColumn.ColumnName.TrimAlias()}]" + $" = " + $"[{SourceTable.GetAlias()}].[{SourceColumn.ColumnName}]"); return(new QueryFactoryResult(joinBuilder.ToString())); }
/// <summary> /// Gets or sets a value indicating whether [trim input]. /// </summary> /// <value> /// <c>true</c> if [trim input]; otherwise, <c>false</c>. /// </value> /* * public List<String> NoDataWillCards * { * get * { * if (!_noDataWillCards.Any()) * { * return DefaultNoDataWillcads; * } * return _noDataWillCards; * } * set * { * if (value.SequenceEqual(DefaultNoDataWillcads)) * { * return; * } * _noDataWillCards = value; * } * }*/ /// <summary> /// Performs a test on /// </summary> /// <param name="sourceTable">The source table.</param> /// <param name="type">The type.</param> /// <param name="i">The i.</param> /// <returns></returns> public SourceTableSliceTest MakeSliceTest(SourceTable sourceTable, SourceTableSliceType type, Int32 i = 0) { SourceTableSliceTest output = new SourceTableSliceTest() { format = type }; switch (type) { case SourceTableSliceType.row: output.Values = sourceTable.GetRow(i); break; default: case SourceTableSliceType.column: output.Values = sourceTable.GetColumn(i); break; } // CellContentType contentType = CellContentType.unknown; foreach (var v in output.Values) { output.ValueStats.Assign(DetermineContentType(v)); //if (contentType == CellContentType.unknown) //{ // contentType = t.type; //} else if (contentType != t.type) //{ // if (!t.type.HasFlag(contentType)) // { // output.IsUniformFormat = false; // } //} if (v.IsNullOrEmpty()) { output.IsNoEmptyValue = false; } if (!output.DistinctValues.Contains(v)) { output.DistinctValues.Add(v); } } output.ValueStats.Compute(); output.IsUniformFormat = output.ValueStats.IsUniformFormat(); if (output.DistinctValues.Count < output.Values.Count) { output.IsDistinctValue = false; } return(output); }
public void CleanDestination(SourceTable tbl) { if (tbl == null) throw new ArgumentNullException("tbl"); if (tbl.DestinationObjectClass == null) { Log.InfoFormat("Skipping cleaning of unmapped table [{0}]", tbl.Name); return; } CleanDestination(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName)); }
/// <summary> /// Performs a test on /// </summary> /// <param name="sourceTable">The source table.</param> /// <param name="type">The type.</param> /// <param name="i">The i.</param> /// <returns></returns> public SourceTableSliceTest MakeSliceTest(SourceTable sourceTable, SourceTableSliceType type, Int32 i = 0) { SourceTableSliceTest output = new SourceTableSliceTest() { format = type }; switch (type) { case SourceTableSliceType.row: output.Values = sourceTable.GetRow(i); break; default: case SourceTableSliceType.column: output.Values = sourceTable.GetColumn(i); break; } CellContentType contentType = CellContentType.unknown; List <String> DistinctValues = new List <string>(); foreach (var v in output.Values) { CellContentInfo t = sourceContentAnalysis.DetermineContentType(v); if (contentType == CellContentType.unknown) { contentType = t.type; } else if (contentType != t.type) { if (!t.type.HasFlag(contentType)) { output.IsUniformFormat = false; } } if (v.IsNullOrEmpty()) { output.IsNoEmptyValue = false; } if (!DistinctValues.Contains(v)) { DistinctValues.Add(v); } } if (DistinctValues.Count < output.Values.Count) { output.IsDistinctValue = false; } return(output); }
public void CleanDestination(SourceTable tbl) { if (tbl == null) { throw new ArgumentNullException("tbl"); } if (tbl.DestinationObjectClass == null) { Log.InfoFormat("Skipping cleaning of unmapped table [{0}]", tbl.Name); return; } CleanDestination(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName)); }
public bool Add(SourceTable model) { bool result = false; _repository.Add(model, (item, err) => { if (err == null) { result = true; } }); return(result); }
public bool Update(SourceTable model, int tkey) { bool _result = false; _repository.Update(model, (item, err) => { if (err == null) { _result = true; } }); return(_result); }
public void BulkOfEntitiesConsistent() { using (var dl = new DisposableList()) { // create the source table var sourceTableWrapper = new TableWrapper(TestHelpers.GenUnique(TestConfig.TableName), TestConfig.PrimaryStorageAccount, true); dl.Add(sourceTableWrapper.Delete); // initialize the target table and attach it to the disposable container var targetTable = new TableWrapper(TestHelpers.GenUnique(TestConfig.TableName), TestConfig.SecondaryStorageAccount, true); dl.Add(targetTable.Delete); var tableParamsResolver = new InMemoryTargetTableResolver(); tableParamsResolver.Add(TestHelpers.TableKey, targetTable); // create gateway blob storage var gateBlob = new InMemoryGatewayBlobStore(); // create a gateway queue var gateQueue = new QueueWrapper(TestHelpers.GenUnique("gateq"), TestConfig.GatewayStorageAccount); var gateQueueWriter = new GatewayQueueWriter(gateQueue, gateBlob); var sourceTable = new SourceTable <StarEntity>(sourceTableWrapper, gateQueueWriter, TestHelpers.TableKey); // 100 entities to satisfy TableStorage's batch requirements var entities = TestHelpers.CreateStarEntities(100); sourceTable.Write(entities); // Now verify that the entities were synced to the secondary table storage TableGatewayQueueProcessor gateQueueProcessor = new TableGatewayQueueProcessor( new GatewayQueueReader(gateQueue, gateBlob), tableParamsResolver ); bool result = gateQueueProcessor.ProcessNext(); Assert.IsTrue(result); var targetEntities = targetTable.ReadEntities <StarEntity>() .ToList(); Assert.AreEqual(entities.Count, targetEntities.Count); foreach (var entity in entities) { TestHelpers.AssertEqualStars( entity, targetEntities.First(x => x.RowKey == entity.RowKey) ); } } }
public Translator(SourceTable tbl, IDataReader source, IEnumerable <SourceColumn> srcColumns, Converter[] converter) { if (tbl == null) { throw new ArgumentNullException("tbl"); } if (source == null) { throw new ArgumentNullException("source"); } if (srcColumns == null) { throw new ArgumentNullException("srcColumns"); } //_tbl = tbl; _source = source; _srcColumns = srcColumns.ToArray(); _srcColumnsInfos = _srcColumns.Select( col => new SourceColumnInfo(DbTypeMapper.GetDbTypeForProperty(col.DestinationProperty.Last().GetType()), DbTypeMapper.GetDbType(col.DbType), col.DestinationProperty.Last() is EnumerationProperty, col.DestinationProperty.First() is CompoundObjectProperty)).ToArray(); _converter = converter != null?converter.ToDictionary(c => c.Column) : new Dictionary <SourceColumn, Converter>(); _resultColumnCount = _srcColumns.Length; if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType())) { // TODO: That's a bad hack! _errorColIdx = _resultColumnCount; _resultColumnCount++; } else { _errorColIdx = -1; } foreach (var comp in srcColumns .Where(c => c.DestinationProperty.First() is CompoundObjectProperty) .GroupBy(c => c.DestinationProperty.First())) { foreach (var col in comp) { _compoundObjectSourceColumns[col.Name] = _resultColumnCount; } _resultColumnCount++; } }
public MetaTableDescription ConstructDescription(SourceTable sourceTable, TableExtractionTask task, ITextRender logger) { //if (sourceTable[0, 0].Value.isNullOrEmpty()) //{ // sourceTable[0, 0].Value = "ID"; //} MetaTableDescription metaDescription = null; builderForText reporter = task.score.CurrentEntry().reporter; switch (task.score.executionMode) { case ExtractionTaskEngineMode.Training: SourceTableDescription sourceDesc = sourceContentAnalysis.GetDescription(sourceTable); metaDescription = new imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableDescription(sourceDesc, imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableFormatType.vertical); // task.tableDescription = metaDescription; //task.score.CurrentEntry().metaTableDescription = metaDescription; break; case ExtractionTaskEngineMode.Validation: default: case ExtractionTaskEngineMode.Application: if (task.tableDescription == null) { throw new Exception("Task [" + task.name + "] has no table description set."); } break; } if (folder != null) { String sp = folder.pathFor("UMTC_Construct_" + task.name + "_" + task.score.executionMode.toString() + ".txt"); File.WriteAllText(sp, reporter.GetContent()); } return(metaDescription); }
public void SetEntriesAndLinkToSource(SourceTable source) { Int32 Skip = description.EntrySkipCount; List <List <SourceTableCell> > data = source.GetContentCells(description.format == MetaTableFormatType.vertical); sourceTable = source; for (int i = Skip; i < data.Count; i++) { if (i != description.index_propertyID) { entries.CreateEntry(data[i], true); } } }
private TableMappingDefinition CreateTableMappingDefinition(SourceTable srcTable, DestinationTable destTable, TableMappingConfiguration mappingConfig) { TableMappingDefinition mappingDefinition; if (mappingConfig != null) { mappingDefinition = new TableMappingDefinition(srcTable, destTable, mappingConfig.FieldMappings); } else { mappingDefinition = new TableMappingDefinition(srcTable, destTable); } return(mappingDefinition); }
private static List <string> GetDestinationColumnNames(SourceTable tbl, List <SourceColumn> srcColumns) { var dstColumnNames = srcColumns.Select(c => GetColName(c.DestinationProperty)).ToList(); // Error Col if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType())) { dstColumnNames.Add("MigrationErrors"); } dstColumnNames.AddRange(srcColumns .Where(c => c.DestinationProperty.First() is CompoundObjectProperty) .GroupBy(c => c.DestinationProperty.First()) .Select(grp => GetColName(grp.Key))); return(dstColumnNames); }
private static DataTable GetTables(IImportSession session) { DataTable dt = new(); dt.Columns.Add("source_table", typeof(SourceTable)); dt.Columns.Add("display_name", typeof(string)); dt.Columns.Add("to_be_imported", typeof(bool)); dt.BeginLoadData(); foreach (var(schema, table) in session.TableNames) { var sourceTable = SourceTable.FromTable(schema, table); dt.LoadDataRow(new object[] { sourceTable, sourceTable.DisplayText, false }, true); } dt.EndLoadData(); return(dt); }
public override SourceTable MakeSourceTable(HtmlNode divNode) { var head = divNode.SelectSingleNode(divNode.XPath.add("div[not(*)]", "//")); if (head == null) { head = divNode; } List <HtmlNode> html_tablerows = AdaptiveRowSelection(head, AdaptiveSteps); //divNode.SelectNodesInDepthRange(x => x.Name.Equals(TableSelectionTag, StringComparison.InvariantCultureIgnoreCase), TableSelectionDepthLimit, TableSelectionDepthStart, false); List <TableHtmlSourceRow> html_selected_rows = new List <TableHtmlSourceRow>(); Int32 columns = 0; foreach (HtmlNode row in html_tablerows) { TableHtmlSourceRow sourceRow = SetSourceRow(row); columns = Math.Max(columns, sourceRow.Count); if (sourceRow.Count > 0) { html_selected_rows.Add(sourceRow); } } Int32 rows = html_selected_rows.Count; SourceTable sourceTable = new SourceTable(columns, rows); if (!sourceTable.IsValid) { if (DoThrowException) { throw new ArgumentOutOfRangeException(nameof(columns), "Number of columns and rows can't be less than 1. Specified values: width [" + columns.ToString() + "], height [" + rows.ToString() + "]"); } } for (int i = 0; i < html_selected_rows.Count; i++) { TableHtmlSourceRow row_node = html_selected_rows[i]; for (int j = 0; j < row_node.RowCells.Count; j++) { SetSourceTableCell(sourceTable[j, i], row_node.RowCells[j], divNode.OwnerDocument); } } return(sourceTable); }
private void TableBaseSimpleMigration(SourceTable tbl, Converter[] nullConverter, List <SourceColumn> mappedColumns) { var dstColumnNames = GetDestinationColumnNames(tbl, mappedColumns); var srcColumnNames = mappedColumns.Select(c => c.Name).ToArray(); var tblRef = _src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name); // no fk mapping required using (var srcReader = _src.ReadTableData(tblRef, srcColumnNames)) using (var translator = new Translator(tbl, srcReader, mappedColumns, nullConverter)) { _dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames); WriteLog( tbl.Name, translator.ProcessedRows, tbl.DestinationObjectClass.TableName, translator.ProcessedRows); } }
private SourceTable GetSourceTable(string destTableName, TableMappingConfiguration mappingConfig) { SourceTable sourceTable = null; if (mappingConfig != null) { string mapSourceTableName = mappingConfig.SourceTableName != null ? mappingConfig.SourceTableName : destTableName; sourceTable = _sourceDatabase.GetTable(mapSourceTableName); } else { sourceTable = _sourceDatabase.GetTable(destTableName); } return(sourceTable); }
private void HandleMaxTextUpdate(SqlGenerator sqlGenerator, TableMappingDefinition mappingDefinition, List <string> scriptNames) { Logger.Info("Handling max text"); SourceTable srcTable = mappingDefinition.SourceTable; DestinationTable destTable = mappingDefinition.DestinationTable; string postSqlPackagePath = Path.Combine(_packageOutputPath, CONVERSION_PACKAGE_FOLDER); Logger.Info("Generating script for updating max text fields"); string script = sqlGenerator.GenerateVarCharMaxUpdateScript(); if (!string.IsNullOrEmpty(script)) { var fileName = string.Format("{0}-{1}_TEXT.sql", srcTable.Name, destTable.Name); SaveToFile(postSqlPackagePath, fileName, script); scriptNames.Add(fileName); } }
public Translator(SourceTable tbl, IDataReader source, IEnumerable<SourceColumn> srcColumns, Converter[] converter) { if (tbl == null) throw new ArgumentNullException("tbl"); if (source == null) throw new ArgumentNullException("source"); if (srcColumns == null) throw new ArgumentNullException("srcColumns"); //_tbl = tbl; _source = source; _srcColumns = srcColumns.ToArray(); _srcColumnsInfos = _srcColumns.Select( col => new SourceColumnInfo(DbTypeMapper.GetDbTypeForProperty(col.DestinationProperty.Last().GetType()), DbTypeMapper.GetDbType(col.DbType), col.DestinationProperty.Last() is EnumerationProperty, col.DestinationProperty.First() is CompoundObjectProperty)).ToArray(); _converter = converter != null ? converter.ToDictionary(c => c.Column) : new Dictionary<SourceColumn, Converter>(); _resultColumnCount = _srcColumns.Length; if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType())) { // TODO: That's a bad hack! _errorColIdx = _resultColumnCount; _resultColumnCount++; } else { _errorColIdx = -1; } foreach (var comp in srcColumns .Where(c => c.DestinationProperty.First() is CompoundObjectProperty) .GroupBy(c => c.DestinationProperty.First())) { foreach (var col in comp) { _compoundObjectSourceColumns[col.Name] = _resultColumnCount; } _resultColumnCount++; } }
public void TableBaseMigration(SourceTable tbl, Converter[] converter, Join[] additional_joins) { // ------------------- Argument checks ------------------- if (tbl == null) throw new ArgumentNullException("tbl"); if (tbl.DestinationObjectClass == null) { Log.InfoFormat("Skipping base migration of unmapped table [{0}]", tbl.Name); return; } using (Log.InfoTraceMethodCallFormat("TableBaseMigration", "{0} to {1}", tbl.Name, tbl.DestinationObjectClass.Name)) { // ------------------- Build columns ------------------- var mappedColumns = tbl.SourceColumn .Where(c => c.DestinationProperty.Count > 0) .OrderBy(c => c.Name) .ToList(); // Ref Cols var referringCols = mappedColumns.Where(c => c.References != null).ToList(); // ------------------- Migrate ------------------- if (referringCols.Count == 0 && (additional_joins == null || additional_joins.Length == 0)) { TableBaseSimpleMigration(tbl, converter, mappedColumns); } else { TableBaseComplexMigration(tbl, converter, mappedColumns, referringCols, additional_joins); } } }
private static List<string> GetDestinationColumnNames(SourceTable tbl, List<SourceColumn> srcColumns) { var dstColumnNames = srcColumns.Select(c => GetColName(c.DestinationProperty)).ToList(); // Error Col if (typeof(IMigrationInfo).IsAssignableFrom(tbl.DestinationObjectClass.GetDataType())) { dstColumnNames.Add("MigrationErrors"); } dstColumnNames.AddRange(srcColumns .Where(c => c.DestinationProperty.First() is CompoundObjectProperty) .GroupBy(c => c.DestinationProperty.First()) .Select(grp => GetColName(grp.Key))); return dstColumnNames; }
private void TableBaseComplexMigration(SourceTable tbl, Converter[] converter, List<SourceColumn> mappedColumns, List<SourceColumn> referringCols, Join[] additional_joins) { if (additional_joins != null && additional_joins.Length > 0 && additional_joins.All(j => referringCols .Select(c => c.DestinationProperty.Single()) .OfType<ObjectReferenceProperty>() .Any(orp => j.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName)))) { throw new InvalidOperationException("Unmapped additional joins found"); } // could automatically create needed indices var all_joins = new Dictionary<SourceColumn, Join>(); var root_joins = referringCols .GroupBy(k => k.DestinationProperty.Single()) .SelectMany(referenceGroup => CreateReferenceJoin(referenceGroup, all_joins)) .ToArray(); // Add manual joins IEnumerable<Join> joins; if (additional_joins != null) { joins = root_joins.Union(additional_joins); } else { joins = root_joins; } var srcColumns = mappedColumns .Where(c => c.References == null) .Union( referringCols .GroupBy(k => k.DestinationProperty.Single()) // referring columns cannot be mapped remotely .Select(g => g.First(p => p.References.References == null)) ).ToList(); var srcColumnNames = srcColumns.Select(c => { var orp = c.DestinationProperty.FirstOrDefault() as ObjectReferenceProperty; if (c.References != null) { return new ProjectionColumn("ID", all_joins[c], System.Data.DbType.Int32, c.DestinationProperty.Single().Name); } else if (c.References == null && orp != null) { if (additional_joins != null && additional_joins.Count(i => i.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName)) > 0) { return new ProjectionColumn( "ID", additional_joins.Single(j => j.JoinTableName == _dst.GetTableName(orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.Module.SchemaName, orp.RelationEnd.Parent.GetOtherEnd(orp.RelationEnd).Type.TableName)), System.Data.DbType.Int32, orp.Name); } else if (converter.Any(cnv => cnv.Column.Name == c.Name)) { return new ProjectionColumn(c.Name, ColumnRef.PrimaryTable, c.DestinationProperty.Single().Name); } else { throw new InvalidOperationException(string.Format("No join found for {0}", c)); } } else { return new ProjectionColumn(c.Name, ColumnRef.PrimaryTable, (System.Data.DbType)c.DbType, null); } }).ToList(); var dstColumnNames = GetDestinationColumnNames(tbl, srcColumns); long processedRows; using (var srcReader = _src.ReadJoin(_src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name), srcColumnNames, joins)) using (var translator = new Translator(tbl, srcReader, srcColumns, converter)) { _dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames); processedRows = translator.ProcessedRows; } // count rows in original table, joins should not add or remove rows WriteLog( tbl.Name, _src.CountRows(_src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name)), tbl.DestinationObjectClass.TableName, processedRows); }
private void TableBaseSimpleMigration(SourceTable tbl, Converter[] nullConverter, List<SourceColumn> mappedColumns) { var dstColumnNames = GetDestinationColumnNames(tbl, mappedColumns); var srcColumnNames = mappedColumns.Select(c => c.Name).ToArray(); var tblRef = _src.GetTableName(tbl.StagingDatabase.Schema, tbl.Name); // no fk mapping required using (var srcReader = _src.ReadTableData(tblRef, srcColumnNames)) using (var translator = new Translator(tbl, srcReader, mappedColumns, nullConverter)) { _dst.WriteTableData(_dst.GetTableName(tbl.DestinationObjectClass.Module.SchemaName, tbl.DestinationObjectClass.TableName), translator, dstColumnNames); WriteLog( tbl.Name, translator.ProcessedRows, tbl.DestinationObjectClass.TableName, translator.ProcessedRows); } }
public void TableBaseMigration(SourceTable tbl) { TableBaseMigration(tbl, (Converter[])null, (Join[])null); }
public void TableBaseMigration(SourceTable tbl, params Converter[] converter) { TableBaseMigration(tbl, converter, null); }