public TransformEventData GenerateTransform(IDictionary <string, ScrapySource> sources, ScheduleTransform[] scheduleTransform)
{
TransformEventData generated = new TransformEventData()
{
SourceMapToTransform = new Dictionary <string, List <string> >(),
TransformEvents = new Dictionary <string, List <TransformEvent> >() /// ALL these should be store to cache.
};
foreach (var item in scheduleTransform)
{
List <TransformEvent> transformEvents = new List <TransformEvent>();
foreach (var sourceName in item.MapToSource)
{
var scrapySource = sources[sourceName];
string newGuid = Guid.NewGuid().ToString();
TransformEvent transformEvent = new TransformEvent()
{
FieldDefinitions = item.FieldDefinitions,
GetFrom = scrapySource.SaveTo,
ExportAs = item.ExportAs,
JobId = newGuid,
SaveTo = "Load/" + newGuid.ToMD5Hex(),
MessageId = scrapySource.MessageId,
SourceId = scrapySource.JobId
};
AddSourceMapToTransform(generated.SourceMapToTransform, scrapySource, transformEvent);
transformEvents.Add(transformEvent);
}
generated.TransformEvents[item.Name] = transformEvents;
}
return(generated);
}
public LoadEventData GenerateLoadEvent(TransformEventData eventData, ScheduleLoad scheduleLoad)
{
LoadEventData loadEventData = new LoadEventData();
var providers = scheduleLoad.LoadProviders;
List <string> selectedTransform = scheduleLoad
.LoadMaps
.Select(x => x.FromTransform)
.Distinct()
.ToList();
foreach (var transformKey in selectedTransform)
{
var transforms = eventData.TransformEvents[transformKey];
var loads = scheduleLoad.LoadMaps
.Where(x => x.FromTransform == transformKey);
foreach (var transform in transforms)
{
var jobId = Guid.NewGuid().ToString();
LoadEvent loadEvent = new LoadEvent()
{
JobId = jobId,
LoadProviders = providers,
MessageId = transform.MessageId,
SourceId = transform.SourceId,
TransformId = transform.JobId,
Data = loads.Select(x => new LoadDataNavigator()
{
DataPacket = transform.SaveTo,
Provider = new LoadProviderSelection()
{
Name = x.LoadProvider,
Parameter = x.Parameter /// Parameter Enablement
}
}).ToArray()
};
loadEventData.AddLoadEvent(loadEvent);
loadEventData.TransformToLoadMap[transform.JobId] = jobId;
}
}
return(loadEventData);
}