public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId);
var sourceType = scrapySource.Source.Type;
IExtractor extractor = extractorManager.GetExtrator(sourceType);
await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo);
string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId;
TaskingManager taskingManager = new TaskingManager();
List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey);
if (jobIds != null)
{
foreach (var x in jobIds)
{
KernelMessage transforMsg = new KernelMessage()
{
JobId = x,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
};
taskingManager.AddTask(platformExit.OutRandom(transforMsg));
logger.Debug(JsonConvert.SerializeObject(transforMsg));
}
}
await taskingManager.WhenAll();
}
public async Task ScheduleNew(ScheduleMessage scheduleMessage, PlatformModel platformModel)
{
Dictionary <string, ScrapySource> sourceDict =
SourceGenManager.Instance.GenerateSource(scheduleMessage.Sources, scheduleMessage.MessageId);
TransformEventData transformEventData =
TransformGenManager.Instance.GenerateTransform(sourceDict, scheduleMessage.Transforms);
LoadEventData loadEventData = LoadGenManager.Instance
.GenerateLoadEvent(transformEventData, scheduleMessage.LandingTargets);
MessageIndexer messageIndexer = new MessageIndexer()
{
MessageId = scheduleMessage.MessageId,
MessageName = scheduleMessage.MessageName,
StartTime = DateTime.Now,
SourceJobIds = sourceDict.Values
.Select(x => x.JobId).ToDictionary(x => x, x => 0),
TransformJobIds = transformEventData.TransformEvents
.SelectMany(x => x.Value)
.Select(x => x.JobId)
.ToDictionary(x => x, x => 0),
LoadJobIds = loadEventData.LoadEvents
.Select(x => x.JobId)
.ToDictionary(x => x, x => 0)
};
TaskingManager manager = new TaskingManager();
manager.AddTask(platformModel.CoreCache.StoreAsync(PrefixConst.MESSAGE_JOBs + messageIndexer.MessageId, messageIndexer));
foreach (var srcKey in transformEventData.SourceMapToTransform.Keys)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.SOURCE_TRANSFOR_MAP + srcKey,
transformEventData.SourceMapToTransform[srcKey]));
}
foreach (var transkv in loadEventData.TransformToLoadMap)
{
manager.AddTask(platformModel.CoreCache.StoreStringAsync(
PrefixConst.TRANSFORM_LOAD_MAP + transkv.Key, transkv.Value));
}
foreach (var source in sourceDict.Values)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.SOURCE_META + source.JobId,
source));
}
foreach (var transform in transformEventData.TransformEvents.SelectMany(x => x.Value))
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.TRANSFORM_META + transform.JobId, transform));
}
foreach (var load in loadEventData.LoadEvents)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.LOAD_META + load.JobId, load));
}
await manager.WhenAll();
await PublishSourceJobs(
scheduleMessage.MessageName,
scheduleMessage.MessageId,
platformModel.PlatformExit,
messageIndexer.SourceJobIds.Keys.ToArray());
}
