public Dictionary <string, ExtractedTable> ExportDataFromDatabase(string query, bool includeSoftDependencies, bool excludeUnwantedDependencies)
{
Dictionary <string, ExtractedTable> extractedTables = new Dictionary <string, ExtractedTable>();
//DataTable primaryKeyConstraints = DataReader.GetPrimaryKeyConstraintsFromDatabase(ServerName, DatabaseName);
List <ForeignKeyConstraint> foreignKeyConstraints = new List <ForeignKeyConstraint>(ForeignKeyConstraints);
//IgnoredDependencyLists ignoredDependencyLists = new IgnoredDependencyLists();
if (includeSoftDependencies)
{
foreignKeyConstraints.AddRange(SoftConstraints);
}
// Parse query into separate selects
List <SelectStatement> selectStatementsFromQuery = SqlStatementParser.ParseSelectStatement(query);
foreach (SelectStatement selectStatement in selectStatementsFromQuery)
{
// Execute unmodified select to pull primary keys for new clean select query
DataSet result = DataReader.ExecuteQuery(ServerName, DatabaseName, selectStatement.SelectText);
// Get primary key constraints for current table
List <string> primaryKeyColumnNames = DataReader.GetListOfPrimaryKeyConstraints(selectStatement.PrimaryTableName, PrimaryKeyConstraints).Select(row => row["ColumnName"].ToString()).ToList();
// Get primary keys from query results
Dictionary <string, List <string> > primaryKeysAndValues = DataReader.GetPrimaryKeyValues(selectStatement.PrimaryTableName, result.Tables[0], PrimaryKeyConstraints);
// Create new select using primary keys
string selectUsingPrimaryKeys = SqlStatementParser.NewSelectWhereKeysInValues(selectStatement.PrimaryTableName.FullNameDelimited, primaryKeysAndValues);
// Execute new select
ExtractedTable extractedTable = new ExtractedTable
{
TableName = selectStatement.PrimaryTableName,
PrimaryKeyColumnNames = primaryKeyColumnNames,
DataTable = DataReader.ExecuteQueryUsingForXml(ServerName, DatabaseName, selectUsingPrimaryKeys).Tables[0]
};
// Set primary keys on DataTable
DataParser.SetDataTablePrimaryKey(primaryKeyColumnNames, extractedTable.DataTable);
extractedTable.DataTable.AcceptChanges();
// Add to extracted tables collection
extractedTables.Add(extractedTable.TableName.FullName, extractedTable);
// Get dependencies
List <TableDependency> parentDependencies = DataParser.GetParentDependenciesList(selectStatement.PrimaryTableName, foreignKeyConstraints, true);
List <TableDependency> childDependencies = DataParser.GetChildDependenciesList(selectStatement.PrimaryTableName, foreignKeyConstraints, true);
// Get child dependencies for all parent dependencies
//foreach (TableDependency tableDependency in parentDependencies)
//{
// childDependencies.AddRange(DataParser.GetChildDependenciesList(tableDependency.ParentTableName.Schema, tableDependency.ParentTableName.Name, foreignKeyConstraints, true));
//}
childDependencies = childDependencies.Distinct().ToList();
if (excludeUnwantedDependencies)
{
DataParser.RemoveUnwantedDependencies(childDependencies, IgnoredDependencyLists.ChildDependencies);
}
// Get parent dependencies for all child dependencies
foreach (TableDependency tableDependency in childDependencies)
{
parentDependencies.AddRange(DataParser.GetParentDependenciesList(tableDependency.ChildTableName.Schema, tableDependency.ChildTableName.Name, foreignKeyConstraints, true));
}
parentDependencies = parentDependencies.Distinct().ToList();
if (excludeUnwantedDependencies)
{
DataParser.RemoveUnwantedDependencies(parentDependencies, IgnoredDependencyLists.ParentDependencies);
}
Queue <TableDependency> parentDependencyQueue = new Queue <TableDependency>(parentDependencies);
Queue <TableDependency> childDependencyQueue = new Queue <TableDependency>(childDependencies);
List <TableDependency> failedParentDependencies = new List <TableDependency>();
List <TableDependency> failedChildDependencies = new List <TableDependency>();
while (parentDependencyQueue.Count > 0 || childDependencyQueue.Count > 0)
{
while (parentDependencyQueue.Count > 0)
{
TableDependency tableDependency = parentDependencyQueue.Dequeue();
if (extractedTables.ContainsKey(tableDependency.ChildTableName.FullName))
{
//Write-Host "Retrieving [{foreignKeyConstraint.ConstraintColumnName)] foreign key values" todo
// Get foreign key values, exclude null values
Dictionary <string, List <string> > keyValuesList = DataParser.GetColumnValues(
extractedTables[tableDependency.ChildTableName.FullName].DataTable,
tableDependency.ColumnDependencies,
TableDependencyType.ChildDependency,
TableDependencyType.ParentDependency,
true,
true);
if (keyValuesList.Count > 0)
{
DataTable dataTable = RetrieveTable(ServerName, DatabaseName, tableDependency.ParentTableName, keyValuesList, PrimaryKeyConstraints);
int originalRowCount = 0;
// Add parent table to overall results
if (extractedTables.ContainsKey(tableDependency.ParentTableName.FullName))
{
originalRowCount = extractedTables[tableDependency.ParentTableName.FullName].DataTable.Rows.Count;
extractedTables[tableDependency.ParentTableName.FullName].DataTable.Merge(dataTable);
}
else
{
extractedTables.Add(
tableDependency.ParentTableName.FullName,
new ExtractedTable
{
TableName = tableDependency.ParentTableName,
PrimaryKeyColumnNames = primaryKeyColumnNames,
DataTable = dataTable
}
);
failedParentDependencies.Remove(tableDependency);
}
//if (originalRowCount < extractedTables[tableDependency.ParentTableName.FullName].DataTable.Rows.Count)
//{
// List<TableDependency> tmp = childDependencyQueue.ToList();
// foreach (TableDependency dependency in DataParser.GetChildDependenciesList(tableDependency.ParentTableName.Schema, tableDependency.ParentTableName.Name, foreignKeyConstraints, true))
// {
// tmp.Add(dependency);
// }
// tmp = tmp.Distinct().ToList();
// DataParser.RemoveUnwantedDependencies(tmp, IgnoredDependencyLists.ChildDependencies);
// childDependencyQueue = new Queue<TableDependency>(tmp);
//}
}
else
{
//Write-Host "Table $parentTableNameDelimited not retrieved. $childTableNameDelimited.[$(foreignKeyConstraint.ConstraintColumnName)] values are all DBNull"; todo
}
}
else
{
if (!failedParentDependencies.Contains(tableDependency))
{
failedParentDependencies.Add(tableDependency);
parentDependencyQueue.Enqueue(tableDependency);
}
}
}
while (childDependencyQueue.Count > 0)
{
TableDependency tableDependency = childDependencyQueue.Dequeue();
if (extractedTables.ContainsKey(tableDependency.ParentTableName.FullName))
{
//Write-Host "Retrieving [{foreignKeyConstraint.ConstraintColumnName)] foreign key values" todo
// Get foreign key values, exclude null values
Dictionary <string, List <string> > keyValuesList = DataParser.GetColumnValues(
extractedTables[tableDependency.ParentTableName.FullName].DataTable,
tableDependency.ColumnDependencies,
TableDependencyType.ParentDependency,
TableDependencyType.ChildDependency,
true,
true);
if (keyValuesList.Count > 0)
{
DataTable dataTable = RetrieveTable(ServerName, DatabaseName, tableDependency.ChildTableName, keyValuesList, PrimaryKeyConstraints);
if (dataTable.Rows.Count > 0)
{
int originalRowCount = 0;
// Add parent table to overall results
if (extractedTables.ContainsKey(tableDependency.ChildTableName.FullName))
{
originalRowCount = extractedTables[tableDependency.ChildTableName.FullName].DataTable.Rows.Count;
extractedTables[tableDependency.ChildTableName.FullName].DataTable.Merge(dataTable);
}
else
{
extractedTables.Add(
tableDependency.ChildTableName.FullName,
new ExtractedTable
{
TableName = tableDependency.ChildTableName,
PrimaryKeyColumnNames = primaryKeyColumnNames,
DataTable = dataTable
}
);
}
if (originalRowCount < extractedTables[tableDependency.ChildTableName.FullName].DataTable.Rows.Count)
{
List <TableDependency> tmp = parentDependencyQueue.ToList();
foreach (TableDependency dependency in DataParser.GetParentDependenciesList(tableDependency.ChildTableName.Schema, tableDependency.ChildTableName.Name, foreignKeyConstraints, true))
{
tmp.Add(dependency);
}
tmp = tmp.Distinct().ToList();
DataParser.RemoveUnwantedDependencies(tmp, IgnoredDependencyLists.ParentDependencies);
parentDependencyQueue = new Queue <TableDependency>(tmp);
}
}
else
{
// todo write warning in log
}
}
else
{
//Write-Host "Table $parentTableNameDelimited not retrieved. $childTableNameDelimited.[$(foreignKeyConstraint.ConstraintColumnName)] values are all DBNull"; todo
}
}
else
{
if (!failedChildDependencies.Contains(tableDependency))
{
failedChildDependencies.Add(tableDependency);
childDependencyQueue.Enqueue(tableDependency);
}
}
}
}
}
return(extractedTables);
}
