public TransformEventData GenerateTransform(IDictionary <string, ScrapySource> sources, ScheduleTransform[] scheduleTransform) { TransformEventData generated = new TransformEventData() { SourceMapToTransform = new Dictionary <string, List <string> >(), TransformEvents = new Dictionary <string, List <TransformEvent> >() /// ALL these should be store to cache. }; foreach (var item in scheduleTransform) { List <TransformEvent> transformEvents = new List <TransformEvent>(); foreach (var sourceName in item.MapToSource) { var scrapySource = sources[sourceName]; string newGuid = Guid.NewGuid().ToString(); TransformEvent transformEvent = new TransformEvent() { FieldDefinitions = item.FieldDefinitions, GetFrom = scrapySource.SaveTo, ExportAs = item.ExportAs, JobId = newGuid, SaveTo = "Load/" + newGuid.ToMD5Hex(), MessageId = scrapySource.MessageId, SourceId = scrapySource.JobId }; AddSourceMapToTransform(generated.SourceMapToTransform, scrapySource, transformEvent); transformEvents.Add(transformEvent); } generated.TransformEvents[item.Name] = transformEvents; } return(generated); }
public LoadEventData GenerateLoadEvent(TransformEventData eventData, ScheduleLoad scheduleLoad) { LoadEventData loadEventData = new LoadEventData(); var providers = scheduleLoad.LoadProviders; List <string> selectedTransform = scheduleLoad .LoadMaps .Select(x => x.FromTransform) .Distinct() .ToList(); foreach (var transformKey in selectedTransform) { var transforms = eventData.TransformEvents[transformKey]; var loads = scheduleLoad.LoadMaps .Where(x => x.FromTransform == transformKey); foreach (var transform in transforms) { var jobId = Guid.NewGuid().ToString(); LoadEvent loadEvent = new LoadEvent() { JobId = jobId, LoadProviders = providers, MessageId = transform.MessageId, SourceId = transform.SourceId, TransformId = transform.JobId, Data = loads.Select(x => new LoadDataNavigator() { DataPacket = transform.SaveTo, Provider = new LoadProviderSelection() { Name = x.LoadProvider, Parameter = x.Parameter /// Parameter Enablement } }).ToArray() }; loadEventData.AddLoadEvent(loadEvent); loadEventData.TransformToLoadMap[transform.JobId] = jobId; } } return(loadEventData); }