public IDatastore LoadDataFromSource(DesignerItemBase sourceItem, IDatastore dataStore, ReportProgressMethod reportProgressMethod)
{
IModule sourceObject = objectResolver.GetModule(sourceItem.ID, sourceItem.ModuleDescription.ModuleType);
IConnection connectionObject = objectResolver.GetConnection(sourceItem.ID);
if (connectionObject == null)
{
if (runLog != null)
{
string label = String.IsNullOrEmpty(sourceItem.ItemLabel) ? "-- No Label --" : sourceItem.ItemLabel;
throw new Exception("No connection was selected for the source '" + label + "'.");
}
dataStore = null;
}
ItemLog itemLog = ItemLog.CreateNew(sourceItem.ID, sourceItem.ItemLabel, sourceItem.ModuleDescription.ModuleType.Name, null);
if (runLog != null)
{
var databaseInterface = SqliteWrapper.GetSqliteWrapper(runLog.RunLogPath, sourceItem.ID, sourceItem.ItemLabel);
itemLog.DatabasePath = runLog.RunLogPath + "\\" + databaseInterface.GetDatabaseName();
parentItemLog.SubFlowLogs.Add(itemLog);
}
((IDataSource)sourceObject).LoadData(connectionObject, dataStore, reportProgressMethod);
itemLog.EndTime = DateTime.Now;
return(dataStore);
}
public List <DesignerItemBase> GetPredecessorNodes(DesignerItemBase designerItem)
{
var incomingConnectionSourceIds = GetIncomingConnections(designerItem).Select(t => t.SourceID);
var predecessorNodes = DesignerItems.Where(t => incomingConnectionSourceIds.Contains(t.ID));
return(predecessorNodes.ToList());
}
private ItemWorker InitializeItemWorker(DesignerItemBase designerItem, StepConfigurationBase stepConfiguration)
{
designerItem.State = ItemState.Initialized;
ItemWorker itemWorker = new ItemWorker(designerItem, stepConfiguration);
itemWorker.BackgroundWorker = new System.ComponentModel.BackgroundWorker();
itemWorker.BackgroundWorker.WorkerReportsProgress = true;
itemWorker.BackgroundWorker.DoWork += bgw_DoWork;
itemWorker.BackgroundWorker.ProgressChanged += BackgroundWorker_ProgressChanged;
itemWorker.BackgroundWorker.RunWorkerCompleted += BackgroundWorker_RunWorkerCompleted;
return(itemWorker);
}
public void TraverseUpwards(DesignerItemBase item)
{
if (!ExecutionPlan.Contains(item))
{
ExecutionPlan.Add(item);
}
var predessorItems = flowGraph.GetPredecessorNodes(item);
foreach (var predessorItem in predessorItems)
{
TraverseUpwards(predessorItem);
}
}
public IDatastore WriteToTarget(DesignerItemBase targetItem, IDatastore dataStore, ReportProgressMethod reportProgressMethod)
{
IModule targetModule = objectResolver.GetModule(targetItem.ID, targetItem.ModuleDescription.ModuleType);
IConnection connectionObject = objectResolver.GetConnection(targetItem.ID);
IDatabaseInterface databaseInterface = SqliteWrapper.GetSqliteWrapper(runLog.RunLogPath, targetItem.ID, targetItem.ItemLabel);
ItemLog itemLog = ItemLog.CreateNew(targetItem.ID, targetItem.ItemLabel, targetItem.ModuleDescription.ModuleType.Name, runLog.RunLogPath + "\\" + databaseInterface.GetDatabaseName());
((IDataTarget)targetModule).WriteData(connectionObject, databaseInterface, dataStore, reportProgressMethod);
itemLog.EndTime = DateTime.Now;
parentItemLog.SubFlowLogs.Add(itemLog);
return(dataStore);
}
public IDatastore ExecuteDesignerItem(DesignerItemBase designerItem, List <IDatastore> dataStores, ReportProgressMethod reportProgressMethod)
{
switch (designerItem.ModuleDescription.Attributes.ModuleType)
{
case ModuleType.Source:
return(LoadDataFromSource(designerItem, dataStores.First(), reportProgressMethod));
case ModuleType.Transformation:
return(TransformData(designerItem, dataStores, reportProgressMethod));
case ModuleType.Target:
return(WriteToTarget(designerItem, dataStores.First(), reportProgressMethod));
default:
throw new Exception("ModuleType " + designerItem.ModuleDescription.Attributes.ModuleType + " has no execution implemented");
}
}
public void ExecuteItem(DesignerItemBase item, RunLog runLog)
{
var incomingConnections = FlowGraph.GetIncomingConnections(item);
var outgoingConnections = FlowGraph.GetOutgoingConnections(item);
List <IDatastore> dataStores = new List <IDatastore>();
if (item.ModuleDescription.Attributes.ModuleType == ModuleType.Source)
{
var dataStore = InitializeDatastore();
dataStores.Add(dataStore);
}
else
{
foreach (var incomingConnection in incomingConnections.OrderBy(t => t.SinkConnectorName))
{
dataStores.Add(DataStreams[incomingConnection]);
DataStreams.Remove(incomingConnection);
}
}
var returnedDataStore = moduleExecution.ExecuteDesignerItem(item, dataStores, ReportProgressMethod);
for (int i = 0; i < outgoingConnections.Count; i++)
{
if (i == 0)
{
DataStreams.Add(outgoingConnections[i], returnedDataStore);
}
else
{
var targetDatastore = InitializeDatastore();
DatastoreHelper.CopyDatastore(returnedDataStore, targetDatastore);
DataStreams.Add(outgoingConnections[i], targetDatastore);
}
}
}
public IDatastore TransformData(DesignerItemBase transformationItem, List <IDatastore> dataStores, ReportProgressMethod reportProgressMethod)
{
var transformationItemConfiguration = objectResolver.LoadItemConfiguration(transformationItem.ID) as TransformationConfiguration;
IConnection transformationConnectionObject = null;
if (transformationItemConfiguration.SelectedConnectionConfigurationId.Equals(Guid.Empty) == false)
{
transformationConnectionObject = objectResolver.GetConnection(transformationItemConfiguration.SelectedConnectionConfigurationId);
}
foreach (var dataStore in dataStores)
{
dataStore.ApplyFilter(transformationItemConfiguration.DataFilter);
}
IModule transformationObject = objectResolver.GetModule(transformationItem.ID, transformationItem.ModuleDescription.ModuleType);
if (transformationObject is IDataTransformation)
{
((IDataTransformation)transformationObject).TransformData(transformationConnectionObject, null, dataStores.First(), reportProgressMethod);
return(dataStores.First());
}
else if (transformationObject is IDataMerge)
{
if (dataStores.Count != 2)
{
throw new Exception("Make sure to connect exactly two incoming datasources on JoinRecords transformations!");
}
var mergedStore = ((IDataMerge)transformationObject).TransformData(transformationConnectionObject, null, dataStores[0], dataStores[1], reportProgressMethod);
return(mergedStore);
}
else
{
throw new Exception("Tranformation does not implement a valid Tranformation-Interface!");
}
}
public ItemWorker(DesignerItemBase designerItem, StepConfigurationBase configuration)
{
this.DesignerItem = designerItem;
this.Configuration = configuration;
}
public int SaveDesignerItem(DesignerItemBase designerItem)
{
return(SaveItem(designerItem));
}
public List <ConnectionBase> GetOutgoingConnections(DesignerItemBase designerItem)
{
var outgoingConnections = DesignerConnections.Where(t => t.SourceID == designerItem.ID);
return(outgoingConnections.ToList());
}
public List <ConnectionBase> GetIncomingConnections(DesignerItemBase designerItem)
{
var incomingConnections = DesignerConnections.Where(t => t.SinkID == designerItem.ID);
return(incomingConnections.ToList());
}