public async Task RunJob(ScriptJob job)
{
S3Object scriptFileObj = null;
var scheduler = componentContext.Resolve <ScriptJobScheduler>();
var s3 = componentContext.Resolve <AWSS3API>();
var options = componentContext.Resolve <AWSScrapeJobOptions>();
scriptFileObj = job.Script.ParseS3URI();
if (!await s3.FileExists(scriptFileObj.Key, scriptFileObj.BucketName))
{
throw new ScrapingException($"No Job Script is found in target S3 location");
}
BrowserScripts browserScripts = null;
if (options.TestMode)
{
var filename = options.TestScriptMapping[job.Script];
if (filename.StartsWith("."))
{
filename = $"{AppContext.BaseDirectory}/{filename}";
}
browserScripts = JsonConvert.DeserializeObject <BrowserScripts>(File.ReadAllText(filename));
}
else
{
browserScripts = await s3.ReadFromJson <BrowserScripts>(scriptFileObj.Key, scriptFileObj.BucketName);
}
// run each scripts
Scripts = browserScripts;
if (browserScripts.Data == null)
{
browserScripts.Data = new Dictionary <string, string>();
}
// load the JObject payload data from the messsage
Jsons = new Dictionary <string, JObject>();
if (job.Payload != null)
{
foreach (var property in job.Payload.Properties())
{
if (property.HasValues)
{
// we will store JObject in Jsons and key-value pairs in Dictionary
switch (property.Value.Type)
{
case JTokenType.Object:
Jsons.Add(property.Name, property.Value <JObject>());
break;
case JTokenType.String:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Integer:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Float:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Boolean:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Date:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
default:
Console.WriteLine($"Payload Property {property.Name} Ignored. Property type {property.Value.Type} is not supported!");
break;
}
}
}
}
foreach (var key in Scripts.Run)
{
if (browserScripts.Data.ContainsKey("success"))
{
browserScripts.Data["success"] = "false";
}
else
{
browserScripts.Data.Add("success", "false");
}
RunScript(key, browserScripts.Data);
switch (browserScripts.Data["success"])
{
case "true":
Console.WriteLine($"Job Succeeded: {job.Script} {job.Job} -> {key}");
await scheduler.CompleteScriptJob(new ScriptJobMessage()
{
Job = job,
ReceiptHandle = null
}, true);
break;
case "false":
Console.WriteLine($"Job Failed: {job.Script} {job.Job} -> {key}");
break;
}
}
if (options.MultipleJobs)
{
Console.WriteLine($"MultipleJobs Mode. Try another job now:");
}
}
public async Task <bool> RunQueueJob()
{
var scheduler = componentContext.Resolve <ScriptJobScheduler>();
var s3 = componentContext.Resolve <AWSS3API>();
var options = componentContext.Resolve <AWSScrapeJobOptions>();
Func <Task> shutdown = async() =>
{
if (options.ShutdownEC2)
{
Console.WriteLine($"Try to Shut down EC2 Instance:");
var ec2_id = Environment.GetEnvironmentVariable(EC2_ID);
if (!string.IsNullOrEmpty(ec2_id))
{
var ec2 = componentContext.Resolve <AWSEC2API>();
Console.WriteLine($"Shutting down EC2 Instance {ec2_id}...");
await ec2.StopByIds(new List <string>() { ec2_id });
// wait stop to run
Thread.Sleep(3000);
}
else
{
Console.WriteLine($"EC2_ID not found in Environment Variable.");
}
}
};
do
{
ScriptJob job = null;
ScriptJobMessage message = null;
S3Object scriptFileObj = null;
while (job == null)
{
message = await scheduler.GetScriptJob();
if (message == null)
{
Console.WriteLine($"No ScriptJob Found. Exit Scrape Engine.");
await shutdown();
return(false);
}
// run the job
job = message.Job;
try
{
if (options.TestMode)
{
if (options.TestScriptMapping.ContainsKey(job.Script))
{
var filename = options.TestScriptMapping[job.Script];
if (filename.StartsWith("."))
{
filename = $"{AppContext.BaseDirectory}/{filename}";
}
if (!File.Exists(filename))
{
Console.WriteLine($"Local Mapping File for Script '{job.Script}' Dost Not Exist in File System. Job is about to Fail.");
}
}
else
{
Console.WriteLine($"No Local Mapping Found for Script '{job.Script}'. Job is about to Fail.");
Debugger.Break();
await scheduler.CompleteScriptJob(message, false);
continue;
}
}
else
{
scriptFileObj = job.Script.ParseS3URI();
if (!await s3.FileExists(scriptFileObj.Key, scriptFileObj.BucketName))
{
await scheduler.CompleteScriptJob(message, false);
continue;
}
}
}
catch (Exception ex)
{
await scheduler.CompleteScriptJob(message, false);
continue;
}
}
// read the script
BrowserScripts browserScripts = null;
if (options.TestMode)
{
var filename = options.TestScriptMapping[job.Script];
if (filename.StartsWith("."))
{
filename = $"{AppContext.BaseDirectory}/{filename}";
}
browserScripts = JsonConvert.DeserializeObject <BrowserScripts>(File.ReadAllText(filename));
}
else
{
browserScripts = await s3.ReadFromJson <BrowserScripts>(scriptFileObj.Key, scriptFileObj.BucketName);
}
// run each scripts
Scripts = browserScripts;
if (browserScripts.Data == null)
{
browserScripts.Data = new Dictionary <string, string>();
}
// load the JObject payload data from the messsage
Jsons = new Dictionary <string, JObject>();
if (job.Payload != null)
{
foreach (var property in job.Payload.Properties())
{
if (property.HasValues)
{
// we will store JObject in Jsons and key-value pairs in Dictionary
switch (property.Value.Type)
{
case JTokenType.Object:
Jsons.Add(property.Name, property.Value <JObject>());
break;
case JTokenType.String:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Integer:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Float:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Boolean:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
case JTokenType.Date:
browserScripts.Data.Add(property.Name, property.Value.Value <string>());
break;
default:
Console.WriteLine($"Payload Property {property.Name} Ignored. Property type {property.Value.Type} is not supported!");
break;
}
}
}
}
foreach (var key in Scripts.Run)
{
if (browserScripts.Data.ContainsKey("success"))
{
browserScripts.Data["success"] = "false";
}
else
{
browserScripts.Data.Add("success", "false");
}
RunScript(key, browserScripts.Data);
switch (browserScripts.Data["success"])
{
case "true":
Console.WriteLine($"Job Succeeded: {job.Script} {job.Job} -> {key}");
scheduler.CompleteScriptJob(message, true).Wait();
break;
case "false":
Console.WriteLine($"Job Failed: {job.Script} {job.Job} -> {key}");
scheduler.CompleteScriptJob(message, false).Wait();
break;
}
}
if (options.MultipleJobs)
{
Console.WriteLine($"MultipleJobs Mode. Try another job now:");
}
} while (options.MultipleJobs);
await shutdown();
return(true);
}