public TransformIntegrationTests()
{
IStorage storage = StorageFactory.Factory.GetLocalStorage(ConstVariable.ApplicationPath);
mockHtmlData = storage.GetString("MockData/Fundamental/Transform/MockTableData.html");
kernelMessage = storage.GetString("MockData/Fundamental/Transform/kernelMessage.json");
datafromCache = storage.GetString("MockData/Fundamental/Transform/httpTransform.json");
result = storage.GetString("MockData/Fundamental/Transform/transformResult.txt");
coreCache = Mock.Of <ICache>();
coreStorage = Mock.Of <IStorage>();
Mock.Get(coreCache)
.Setup(x => x.RestoreAsync <TransformEvent>(It.IsAny <string>()))
.Returns(Task.FromResult(JsonConvert.DeserializeObject <TransformEvent>(datafromCache)));
Mock.Get(coreStorage)
.Setup(x => x.GetStringAsync(It.IsAny <string>()))
.Returns(Task.FromResult(mockHtmlData));
Mock.Get(coreStorage)
.Setup(x => x.WriteStream(It.IsAny <Stream>(), It.IsAny <String>()))
.Returns(new Func <Stream, string, Task>(AssertTransformResult));
this.platformExit = Mock.Of <IPlatformExit>();
Mock.Get(platformExit).Setup(x => x.OutRandom(It.IsAny <KernelMessage>()))
.Returns(Task.CompletedTask);
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId);
var sourceType = scrapySource.Source.Type;
IExtractor extractor = extractorManager.GetExtrator(sourceType);
await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo);
string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId;
TaskingManager taskingManager = new TaskingManager();
List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey);
if (jobIds != null)
{
foreach (var x in jobIds)
{
KernelMessage transforMsg = new KernelMessage()
{
JobId = x,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
};
taskingManager.AddTask(platformExit.OutRandom(transforMsg));
logger.Debug(JsonConvert.SerializeObject(transforMsg));
}
}
await taskingManager.WhenAll();
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage =
JsonConvert.DeserializeObject <KernelMessage>(
Encoding.UTF8.GetString(processMessage));
ScheduleMessage scheduleMessage = await RestoreMeta(kernelMessage);
var scheduler = ScheduleManager.Manager
.GetScheduler(scheduleMessage.Scheduler);
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
TransformEvent transformEvent = await CoreCache.RestoreAsync <TransformEvent>(PrefixConst.TRANSFORM_META + kernelMessage.JobId);
if (transformEvent != null)
{
try
{
TransformDataSet transformDataSet = new TransformDataSet();
string data = await coreStorage.GetStringAsync(transformEvent.GetFrom);
foreach (var def in transformEvent.FieldDefinitions)
{
ContextData contextData = new ContextData()
{
ContentText = data
};
var convertorSquence = def.ConvertorNavigators.Select(x => ConvertorManager.GetConvertor(x));
foreach (var convertor in convertorSquence)
{
contextData = convertor.Convert(contextData);
}
TransformFieldWithValue transformFieldWithValue = new TransformFieldWithValue();
PackageTransformFieldWithValue(transformFieldWithValue, contextData);
transformFieldWithValue.Name = def.Name;
transformFieldWithValue.Title = def.Title;
transformDataSet.FieldValues[def.Name] = transformFieldWithValue;
}
using (Stream serialzedStream = await transformDataSet.SerialzeToStream(transformEvent.ExportAs))
{
await coreStorage.WriteStream(
serialzedStream,
transformEvent.SaveTo);
}
string loadKey = await CoreCache.RestoreStringAsync(PrefixConst.TRANSFORM_LOAD_MAP + transformEvent.JobId);
await platformExit.OutRandom(new KernelMessage()
{
JobId = loadKey,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
});
}
catch (Exception ex)
{
logger.Warn(ex);
}
}
else
{
//TODO: Debug
logger.Debug("TransformEvent is Null:" + PrefixConst.TRANSFORM_META + kernelMessage.JobId);
}
}
private async Task PublishSourceJobs(string name, string messageId, IPlatformExit exit, params string[] sourceJobs)
{
foreach (var item in sourceJobs)
{
KernelMessage kernelMessage = new KernelMessage()
{
JobId = item,
MessageId = messageId,
MessageName = name
};
await exit.OutRandom(kernelMessage);
}
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
LoadEvent loadEvent = await CoreCache.RestoreAsync <LoadEvent>(PrefixConst.LOAD_META + kernelMessage.JobId);
if (loadEvent != null)
{
var loadProviderNavs = loadEvent.LoadProviders.ToDictionary(x => x.Name, x => x);
foreach (var dataProvider in loadEvent.Data)
{
var loadProviderNav = loadProviderNavs[dataProvider.Provider.Name];
ILoadProvider loadProvider = this.LoadProviderManager.GetLoadProvider(loadProviderNav.Type, loadProviderNav.Name, loadProviderNav.Context.ToString());
try
{
await dataStorage.ReadAsStream(dataProvider.DataPacket, s =>
{
return(loadProvider.Load(s,
new LoadContext()
{
Parameter = dataProvider.Provider.Parameter,
PlatformModel = new PlatformModel()
{
CoreCache = CoreCache,
PlatformExit = platformExit
},
LoadEvent = loadEvent
}));
});
}
catch (Exception ex)
{
logger.Error(ex);
}
}
}
else
{
logger.Warn("Received no-meta id:" + PrefixConst.LOAD_META + kernelMessage.JobId);
}
}
public SourceIntergationTests()
{
var storage = StorageFactory.Factory.GetLocalStorage(ConstVariable.ApplicationPath);
httpSourceDemoString = storage.GetString("MockData/Fundamental/Extract/httpsourcedemo.json");
scrapySource = JsonConvert.DeserializeObject <ScrapySource>(httpSourceDemoString);
cache = Mock.Of <ICache>();
extractorManager = Mock.Of <IExtractorManager>();
var extractor = Mock.Of <IExtractor>();
platformExit = Mock.Of <IPlatformExit>();
Mock.Get(cache)
.Setup(x => x.RestoreAsync <ScrapySource>(It.IsAny <string>()))
.Returns(Task.FromResult(scrapySource));
Mock.Get(cache)
.Setup(x => x.RestoreAsync <List <string> >(It.IsAny <string>()))
.Returns(Task.FromResult(new List <string>()
{
"a", "b"
}));
Mock.Get(extractorManager)
.Setup(x => x.GetExtrator(It.IsAny <string>()))
.Returns(extractor);
Mock.Get(extractor).Setup(x => x.ExtractTarget(It.IsAny <string>(), It.IsAny <string>()))
.Returns((string a, string b) =>
{
Assert.Equal(scrapySource.Source.Parameters.ToString(), a);
Assert.Equal(scrapySource.SaveTo, b);
return(Task.CompletedTask);
});
Mock.Get(platformExit).Setup(x => x.OutRandom(It.IsAny <PlatformMessage>()))
.Returns(Task.CompletedTask);
}
public IScheduler GetDefaultScheduler(IPlatformExit platformExit, ICache coreCache)
{
return(new DefaultSchueduler(platformExit, coreCache));
}
public WorkingProcessor(IWorkingMessageProcessor workProcessor, IPlatformExit exit)
{
WorkProcessor = workProcessor;
this.exit = exit;
}
public ScheduleIntegration(IStorage coreStorage, ICache coreCache, IPlatformExit platformExit)
{
this.coreStorage = coreStorage;
this.coreCache = coreCache;
this.platformExit = platformExit;
}
public ScarpyController(ICache cache, IMessageQueue messageQueue)
{
this.cache = cache;
this.messageQueue = messageQueue;
this.platformExit = new MessageQueuePlatformExit(messageQueue);
}
public DefaultSchueduler(IPlatformExit exit, ICache corecache)
{
this.exit = exit;
this.coreCache = corecache;
}
