public TransformIntegrationTests() { IStorage storage = StorageFactory.Factory.GetLocalStorage(ConstVariable.ApplicationPath); mockHtmlData = storage.GetString("MockData/Fundamental/Transform/MockTableData.html"); kernelMessage = storage.GetString("MockData/Fundamental/Transform/kernelMessage.json"); datafromCache = storage.GetString("MockData/Fundamental/Transform/httpTransform.json"); result = storage.GetString("MockData/Fundamental/Transform/transformResult.txt"); coreCache = Mock.Of <ICache>(); coreStorage = Mock.Of <IStorage>(); Mock.Get(coreCache) .Setup(x => x.RestoreAsync <TransformEvent>(It.IsAny <string>())) .Returns(Task.FromResult(JsonConvert.DeserializeObject <TransformEvent>(datafromCache))); Mock.Get(coreStorage) .Setup(x => x.GetStringAsync(It.IsAny <string>())) .Returns(Task.FromResult(mockHtmlData)); Mock.Get(coreStorage) .Setup(x => x.WriteStream(It.IsAny <Stream>(), It.IsAny <String>())) .Returns(new Func <Stream, string, Task>(AssertTransformResult)); this.platformExit = Mock.Of <IPlatformExit>(); Mock.Get(platformExit).Setup(x => x.OutRandom(It.IsAny <KernelMessage>())) .Returns(Task.CompletedTask); }
public async Task Process(byte[] processMessage, IPlatformExit platformExit) { KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage)); ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId); var sourceType = scrapySource.Source.Type; IExtractor extractor = extractorManager.GetExtrator(sourceType); await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo); string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId; TaskingManager taskingManager = new TaskingManager(); List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey); if (jobIds != null) { foreach (var x in jobIds) { KernelMessage transforMsg = new KernelMessage() { JobId = x, MessageId = kernelMessage.MessageId, MessageName = kernelMessage.MessageName }; taskingManager.AddTask(platformExit.OutRandom(transforMsg)); logger.Debug(JsonConvert.SerializeObject(transforMsg)); } } await taskingManager.WhenAll(); }
public async Task Process(byte[] processMessage, IPlatformExit platformExit) { KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>( Encoding.UTF8.GetString(processMessage)); ScheduleMessage scheduleMessage = await RestoreMeta(kernelMessage); var scheduler = ScheduleManager.Manager .GetScheduler(scheduleMessage.Scheduler); }
public async Task Process(byte[] processMessage, IPlatformExit platformExit) { KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage)); TransformEvent transformEvent = await CoreCache.RestoreAsync <TransformEvent>(PrefixConst.TRANSFORM_META + kernelMessage.JobId); if (transformEvent != null) { try { TransformDataSet transformDataSet = new TransformDataSet(); string data = await coreStorage.GetStringAsync(transformEvent.GetFrom); foreach (var def in transformEvent.FieldDefinitions) { ContextData contextData = new ContextData() { ContentText = data }; var convertorSquence = def.ConvertorNavigators.Select(x => ConvertorManager.GetConvertor(x)); foreach (var convertor in convertorSquence) { contextData = convertor.Convert(contextData); } TransformFieldWithValue transformFieldWithValue = new TransformFieldWithValue(); PackageTransformFieldWithValue(transformFieldWithValue, contextData); transformFieldWithValue.Name = def.Name; transformFieldWithValue.Title = def.Title; transformDataSet.FieldValues[def.Name] = transformFieldWithValue; } using (Stream serialzedStream = await transformDataSet.SerialzeToStream(transformEvent.ExportAs)) { await coreStorage.WriteStream( serialzedStream, transformEvent.SaveTo); } string loadKey = await CoreCache.RestoreStringAsync(PrefixConst.TRANSFORM_LOAD_MAP + transformEvent.JobId); await platformExit.OutRandom(new KernelMessage() { JobId = loadKey, MessageId = kernelMessage.MessageId, MessageName = kernelMessage.MessageName }); } catch (Exception ex) { logger.Warn(ex); } } else { //TODO: Debug logger.Debug("TransformEvent is Null:" + PrefixConst.TRANSFORM_META + kernelMessage.JobId); } }
private async Task PublishSourceJobs(string name, string messageId, IPlatformExit exit, params string[] sourceJobs) { foreach (var item in sourceJobs) { KernelMessage kernelMessage = new KernelMessage() { JobId = item, MessageId = messageId, MessageName = name }; await exit.OutRandom(kernelMessage); } }
public async Task Process(byte[] processMessage, IPlatformExit platformExit) { KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage)); LoadEvent loadEvent = await CoreCache.RestoreAsync <LoadEvent>(PrefixConst.LOAD_META + kernelMessage.JobId); if (loadEvent != null) { var loadProviderNavs = loadEvent.LoadProviders.ToDictionary(x => x.Name, x => x); foreach (var dataProvider in loadEvent.Data) { var loadProviderNav = loadProviderNavs[dataProvider.Provider.Name]; ILoadProvider loadProvider = this.LoadProviderManager.GetLoadProvider(loadProviderNav.Type, loadProviderNav.Name, loadProviderNav.Context.ToString()); try { await dataStorage.ReadAsStream(dataProvider.DataPacket, s => { return(loadProvider.Load(s, new LoadContext() { Parameter = dataProvider.Provider.Parameter, PlatformModel = new PlatformModel() { CoreCache = CoreCache, PlatformExit = platformExit }, LoadEvent = loadEvent })); }); } catch (Exception ex) { logger.Error(ex); } } } else { logger.Warn("Received no-meta id:" + PrefixConst.LOAD_META + kernelMessage.JobId); } }
public SourceIntergationTests() { var storage = StorageFactory.Factory.GetLocalStorage(ConstVariable.ApplicationPath); httpSourceDemoString = storage.GetString("MockData/Fundamental/Extract/httpsourcedemo.json"); scrapySource = JsonConvert.DeserializeObject <ScrapySource>(httpSourceDemoString); cache = Mock.Of <ICache>(); extractorManager = Mock.Of <IExtractorManager>(); var extractor = Mock.Of <IExtractor>(); platformExit = Mock.Of <IPlatformExit>(); Mock.Get(cache) .Setup(x => x.RestoreAsync <ScrapySource>(It.IsAny <string>())) .Returns(Task.FromResult(scrapySource)); Mock.Get(cache) .Setup(x => x.RestoreAsync <List <string> >(It.IsAny <string>())) .Returns(Task.FromResult(new List <string>() { "a", "b" })); Mock.Get(extractorManager) .Setup(x => x.GetExtrator(It.IsAny <string>())) .Returns(extractor); Mock.Get(extractor).Setup(x => x.ExtractTarget(It.IsAny <string>(), It.IsAny <string>())) .Returns((string a, string b) => { Assert.Equal(scrapySource.Source.Parameters.ToString(), a); Assert.Equal(scrapySource.SaveTo, b); return(Task.CompletedTask); }); Mock.Get(platformExit).Setup(x => x.OutRandom(It.IsAny <PlatformMessage>())) .Returns(Task.CompletedTask); }
public IScheduler GetDefaultScheduler(IPlatformExit platformExit, ICache coreCache) { return(new DefaultSchueduler(platformExit, coreCache)); }
public WorkingProcessor(IWorkingMessageProcessor workProcessor, IPlatformExit exit) { WorkProcessor = workProcessor; this.exit = exit; }
public ScheduleIntegration(IStorage coreStorage, ICache coreCache, IPlatformExit platformExit) { this.coreStorage = coreStorage; this.coreCache = coreCache; this.platformExit = platformExit; }
public ScarpyController(ICache cache, IMessageQueue messageQueue) { this.cache = cache; this.messageQueue = messageQueue; this.platformExit = new MessageQueuePlatformExit(messageQueue); }
public DefaultSchueduler(IPlatformExit exit, ICache corecache) { this.exit = exit; this.coreCache = corecache; }