public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId);
var sourceType = scrapySource.Source.Type;
IExtractor extractor = extractorManager.GetExtrator(sourceType);
await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo);
string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId;
TaskingManager taskingManager = new TaskingManager();
List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey);
if (jobIds != null)
{
foreach (var x in jobIds)
{
KernelMessage transforMsg = new KernelMessage()
{
JobId = x,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
};
taskingManager.AddTask(platformExit.OutRandom(transforMsg));
logger.Debug(JsonConvert.SerializeObject(transforMsg));
}
}
await taskingManager.WhenAll();
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
TransformEvent transformEvent = await CoreCache.RestoreAsync <TransformEvent>(PrefixConst.TRANSFORM_META + kernelMessage.JobId);
if (transformEvent != null)
{
try
{
TransformDataSet transformDataSet = new TransformDataSet();
string data = await coreStorage.GetStringAsync(transformEvent.GetFrom);
foreach (var def in transformEvent.FieldDefinitions)
{
ContextData contextData = new ContextData()
{
ContentText = data
};
var convertorSquence = def.ConvertorNavigators.Select(x => ConvertorManager.GetConvertor(x));
foreach (var convertor in convertorSquence)
{
contextData = convertor.Convert(contextData);
}
TransformFieldWithValue transformFieldWithValue = new TransformFieldWithValue();
PackageTransformFieldWithValue(transformFieldWithValue, contextData);
transformFieldWithValue.Name = def.Name;
transformFieldWithValue.Title = def.Title;
transformDataSet.FieldValues[def.Name] = transformFieldWithValue;
}
using (Stream serialzedStream = await transformDataSet.SerialzeToStream(transformEvent.ExportAs))
{
await coreStorage.WriteStream(
serialzedStream,
transformEvent.SaveTo);
}
string loadKey = await CoreCache.RestoreStringAsync(PrefixConst.TRANSFORM_LOAD_MAP + transformEvent.JobId);
await platformExit.OutRandom(new KernelMessage()
{
JobId = loadKey,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
});
}
catch (Exception ex)
{
logger.Warn(ex);
}
}
else
{
//TODO: Debug
logger.Debug("TransformEvent is Null:" + PrefixConst.TRANSFORM_META + kernelMessage.JobId);
}
}
private async Task PublishSourceJobs(string name, string messageId, IPlatformExit exit, params string[] sourceJobs)
{
foreach (var item in sourceJobs)
{
KernelMessage kernelMessage = new KernelMessage()
{
JobId = item,
MessageId = messageId,
MessageName = name
};
await exit.OutRandom(kernelMessage);
}
}