public static async Task <int> QueryGuru(
[ActivityTrigger] DurableActivityContext ctx,
TraceWriter log)
{
var arguments = ctx.GetInput <Tuple <string, string> >();
var searchargs = arguments.Item1.Split(':');
if (searchargs.Length != 2)
{
throw new ArgumentException("Activity.QueryGuru: Expected search query to be in format <query>:<type>");
}
var query = searchargs[0];
var type = searchargs[1];
var cookies = CookieConverter.DecodeCookie(arguments.Item2);
var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });
var page = await browser.NewPageAsync();
await page.SetCookieAsync(cookies);
var search = new UIAction.NavigateSearch(query, type);
var pages = await search.RunAsync(page);
browser.Disconnect();
return(pages);
}
public static async Task <string> Authenticate(
[ActivityTrigger] DurableActivityContext ctx,
TraceWriter log)
{
var credentials = ctx.GetInput <string>();
var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });
var page = await browser.NewPageAsync();
var login = new UIAction.Login();
CookieParam[] cookies = null;
cookies = await login.RunAsync(page, log);
// page is closed implicitly
try
{
browser.Disconnect();
}
catch { }
if (cookies != null)
{
return(CookieConverter.EncodeCookie(cookies));
}
throw new Exception("Failed to authenticate.");
}
public static async Task <object> NewWorker(
[OrchestrationTrigger] DurableOrchestrationContext ctx,
TraceWriter log
)
{
int retryattempts = 1;
var query = ctx.GetInput <string>();
var retryOptions = new RetryOptions(
firstRetryInterval: TimeSpan.FromSeconds(5),
maxNumberOfAttempts: retryattempts);
var getcookietask = ctx.CallActivityWithRetryAsync <string>("Authenticate", retryOptions, "credentials");
getcookietask.ContinueWith(t =>
{
return("Failed");
}, TaskContinuationOptions.OnlyOnFaulted);
await getcookietask;
var cookiesjsonb64 = getcookietask.Result;
var cookies = CookieConverter.DecodeCookie(cookiesjsonb64);
if (!ctx.IsReplaying)
{
log.Warning($"Successfully retrieved {cookies.Length} cookies.");
}
var querygurutask = ctx.CallActivityWithRetryAsync <int>("QueryGuru", retryOptions, new Tuple <string, string>(query, cookiesjsonb64));
querygurutask.ContinueWith(t =>
{
return("Failed");
}, TaskContinuationOptions.OnlyOnFaulted);
var pages = await querygurutask;
if (!ctx.IsReplaying)
{
log.Warning($"Query successfully returned {pages}.");
}
var workload = await ctx.CallActivityAsync <List <int> >("CreateWorkload", pages);
//log.Warning(ctx.InstanceId);
var worker = ctx.CallSubOrchestratorAsync("WorkerWork", new WorkerWorkArgs {
query = query, cookiestext = cookiesjsonb64, workload = workload, total = workload.Count
});
string result = await ctx.WaitForExternalEvent <string>("Finished");
return(result);
}
public static async Task <WorkerResult> RetrievePageContent(
[ActivityTrigger] DurableActivityContext ctx,
TraceWriter log)
{
var arguments = ctx.GetInput <Tuple <string, string, int, string, int> >();
var searchargs = arguments.Item1.Split(':');
if (searchargs.Length != 2)
{
throw new ArgumentException("Activity.QueryGuru: Expected search query to be in format <query>:<type>");
}
var query = searchargs[0];
var type = searchargs[1];
var cookies = CookieConverter.DecodeCookie(arguments.Item2);
var pagenumber = arguments.Item3;
var config = JsonConvert.DeserializeObject <ConfigSection>(arguments.Item4);
var totalpages = arguments.Item5;
var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });
var page = await browser.NewPageAsync();
await page.SetCookieAsync(cookies);
var search = new UIAction.NavigateSearch(query, type, true);
await search.RunAsync(page);
var collect = new UIAction.NavigateCollect(pagenumber, ctx.InstanceId, ctx.InstanceId, config);
var result = await collect.RunAsync(page);
// page is closed implicitly
try
{
browser.Disconnect();
}
catch { }
var customresult = new WorkerResult {
html = result, suburb = query, totalpages = totalpages, page = pagenumber
};
return(customresult);
}