public async Task ScheduleNew(ScheduleMessage scheduleMessage, PlatformModel platformModel)
{
Dictionary <string, ScrapySource> sourceDict =
SourceGenManager.Instance.GenerateSource(scheduleMessage.Sources, scheduleMessage.MessageId);
TransformEventData transformEventData =
TransformGenManager.Instance.GenerateTransform(sourceDict, scheduleMessage.Transforms);
LoadEventData loadEventData = LoadGenManager.Instance
.GenerateLoadEvent(transformEventData, scheduleMessage.LandingTargets);
MessageIndexer messageIndexer = new MessageIndexer()
{
MessageId = scheduleMessage.MessageId,
MessageName = scheduleMessage.MessageName,
StartTime = DateTime.Now,
SourceJobIds = sourceDict.Values
.Select(x => x.JobId).ToDictionary(x => x, x => 0),
TransformJobIds = transformEventData.TransformEvents
.SelectMany(x => x.Value)
.Select(x => x.JobId)
.ToDictionary(x => x, x => 0),
LoadJobIds = loadEventData.LoadEvents
.Select(x => x.JobId)
.ToDictionary(x => x, x => 0)
};
TaskingManager manager = new TaskingManager();
manager.AddTask(platformModel.CoreCache.StoreAsync(PrefixConst.MESSAGE_JOBs + messageIndexer.MessageId, messageIndexer));
foreach (var srcKey in transformEventData.SourceMapToTransform.Keys)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.SOURCE_TRANSFOR_MAP + srcKey,
transformEventData.SourceMapToTransform[srcKey]));
}
foreach (var transkv in loadEventData.TransformToLoadMap)
{
manager.AddTask(platformModel.CoreCache.StoreStringAsync(
PrefixConst.TRANSFORM_LOAD_MAP + transkv.Key, transkv.Value));
}
foreach (var source in sourceDict.Values)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.SOURCE_META + source.JobId,
source));
}
foreach (var transform in transformEventData.TransformEvents.SelectMany(x => x.Value))
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.TRANSFORM_META + transform.JobId, transform));
}
foreach (var load in loadEventData.LoadEvents)
{
manager.AddTask(platformModel.CoreCache.StoreAsync(
PrefixConst.LOAD_META + load.JobId, load));
}
await manager.WhenAll();
await PublishSourceJobs(
scheduleMessage.MessageName,
scheduleMessage.MessageId,
platformModel.PlatformExit,
messageIndexer.SourceJobIds.Keys.ToArray());
}
public override IObservable <EventMessage> OnPublishAsObservable() => OnEventAsObservable() .Select(_ => EventMessage.Create(EventType.TransformEvent, Component, TransformEventData.Create(TransformEventType)));
