Exemplo n.º 1
0
        public TransformEventData GenerateTransform(IDictionary <string, ScrapySource> sources, ScheduleTransform[] scheduleTransform)
        {
            TransformEventData generated = new TransformEventData()
            {
                SourceMapToTransform = new Dictionary <string, List <string> >(),
                TransformEvents      = new Dictionary <string, List <TransformEvent> >() /// ALL these should be store to cache.
            };

            foreach (var item in scheduleTransform)
            {
                List <TransformEvent> transformEvents = new List <TransformEvent>();
                foreach (var sourceName in item.MapToSource)
                {
                    var            scrapySource   = sources[sourceName];
                    string         newGuid        = Guid.NewGuid().ToString();
                    TransformEvent transformEvent = new TransformEvent()
                    {
                        FieldDefinitions = item.FieldDefinitions,
                        GetFrom          = scrapySource.SaveTo,
                        ExportAs         = item.ExportAs,
                        JobId            = newGuid,
                        SaveTo           = "Load/" + newGuid.ToMD5Hex(),
                        MessageId        = scrapySource.MessageId,
                        SourceId         = scrapySource.JobId
                    };
                    AddSourceMapToTransform(generated.SourceMapToTransform, scrapySource, transformEvent);
                    transformEvents.Add(transformEvent);
                }
                generated.TransformEvents[item.Name] = transformEvents;
            }
            return(generated);
        }
Exemplo n.º 2
0
        public LoadEventData GenerateLoadEvent(TransformEventData eventData, ScheduleLoad scheduleLoad)
        {
            LoadEventData loadEventData     = new LoadEventData();
            var           providers         = scheduleLoad.LoadProviders;
            List <string> selectedTransform = scheduleLoad
                                              .LoadMaps
                                              .Select(x => x.FromTransform)
                                              .Distinct()
                                              .ToList();

            foreach (var transformKey in selectedTransform)
            {
                var transforms = eventData.TransformEvents[transformKey];
                var loads      = scheduleLoad.LoadMaps
                                 .Where(x => x.FromTransform == transformKey);
                foreach (var transform in transforms)
                {
                    var       jobId     = Guid.NewGuid().ToString();
                    LoadEvent loadEvent = new LoadEvent()
                    {
                        JobId         = jobId,
                        LoadProviders = providers,
                        MessageId     = transform.MessageId,
                        SourceId      = transform.SourceId,
                        TransformId   = transform.JobId,
                        Data          = loads.Select(x => new LoadDataNavigator()
                        {
                            DataPacket = transform.SaveTo,
                            Provider   = new LoadProviderSelection()
                            {
                                Name      = x.LoadProvider,
                                Parameter = x.Parameter /// Parameter Enablement
                            }
                        }).ToArray()
                    };
                    loadEventData.AddLoadEvent(loadEvent);
                    loadEventData.TransformToLoadMap[transform.JobId] = jobId;
                }
            }
            return(loadEventData);
        }