public static async Task <int> QueryGuru(
            [ActivityTrigger] DurableActivityContext ctx,
            TraceWriter log)
        {
            var arguments  = ctx.GetInput <Tuple <string, string> >();
            var searchargs = arguments.Item1.Split(':');

            if (searchargs.Length != 2)
            {
                throw new ArgumentException("Activity.QueryGuru: Expected search query to be in format <query>:<type>");
            }

            var query   = searchargs[0];
            var type    = searchargs[1];
            var cookies = CookieConverter.DecodeCookie(arguments.Item2);

            var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });

            var page = await browser.NewPageAsync();

            await page.SetCookieAsync(cookies);

            var search = new UIAction.NavigateSearch(query, type);
            var pages  = await search.RunAsync(page);

            browser.Disconnect();
            return(pages);
        }
示例#2
0
        public static async Task <string> Authenticate(
            [ActivityTrigger] DurableActivityContext ctx,
            TraceWriter log)
        {
            var credentials = ctx.GetInput <string>();
            var browser     = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });

            var page = await browser.NewPageAsync();

            var login = new UIAction.Login();

            CookieParam[] cookies = null;
            cookies = await login.RunAsync(page, log);

            // page is closed implicitly
            try
            {
                browser.Disconnect();
            }
            catch { }

            if (cookies != null)
            {
                return(CookieConverter.EncodeCookie(cookies));
            }

            throw new Exception("Failed to authenticate.");
        }
        public static async Task <object> NewWorker(
            [OrchestrationTrigger] DurableOrchestrationContext ctx,
            TraceWriter log
            )
        {
            int retryattempts = 1;
            var query         = ctx.GetInput <string>();

            var retryOptions = new RetryOptions(
                firstRetryInterval: TimeSpan.FromSeconds(5),
                maxNumberOfAttempts: retryattempts);

            var getcookietask = ctx.CallActivityWithRetryAsync <string>("Authenticate", retryOptions, "credentials");

            getcookietask.ContinueWith(t =>
            {
                return("Failed");
            }, TaskContinuationOptions.OnlyOnFaulted);
            await getcookietask;
            var   cookiesjsonb64 = getcookietask.Result;
            var   cookies        = CookieConverter.DecodeCookie(cookiesjsonb64);

            if (!ctx.IsReplaying)
            {
                log.Warning($"Successfully retrieved {cookies.Length} cookies.");
            }

            var querygurutask = ctx.CallActivityWithRetryAsync <int>("QueryGuru", retryOptions, new Tuple <string, string>(query, cookiesjsonb64));

            querygurutask.ContinueWith(t =>
            {
                return("Failed");
            }, TaskContinuationOptions.OnlyOnFaulted);
            var pages = await querygurutask;

            if (!ctx.IsReplaying)
            {
                log.Warning($"Query successfully returned {pages}.");
            }

            var workload = await ctx.CallActivityAsync <List <int> >("CreateWorkload", pages);

            //log.Warning(ctx.InstanceId);
            var worker = ctx.CallSubOrchestratorAsync("WorkerWork", new WorkerWorkArgs {
                query = query, cookiestext = cookiesjsonb64, workload = workload, total = workload.Count
            });

            string result = await ctx.WaitForExternalEvent <string>("Finished");

            return(result);
        }
示例#4
0
        public static async Task <WorkerResult> RetrievePageContent(
            [ActivityTrigger] DurableActivityContext ctx,
            TraceWriter log)
        {
            var arguments  = ctx.GetInput <Tuple <string, string, int, string, int> >();
            var searchargs = arguments.Item1.Split(':');

            if (searchargs.Length != 2)
            {
                throw new ArgumentException("Activity.QueryGuru: Expected search query to be in format <query>:<type>");
            }

            var query      = searchargs[0];
            var type       = searchargs[1];
            var cookies    = CookieConverter.DecodeCookie(arguments.Item2);
            var pagenumber = arguments.Item3;
            var config     = JsonConvert.DeserializeObject <ConfigSection>(arguments.Item4);
            var totalpages = arguments.Item5;

            var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = Constants.BrowserWSEndpoint });

            var page = await browser.NewPageAsync();

            await page.SetCookieAsync(cookies);

            var search = new UIAction.NavigateSearch(query, type, true);
            await search.RunAsync(page);

            var collect = new UIAction.NavigateCollect(pagenumber, ctx.InstanceId, ctx.InstanceId, config);
            var result  = await collect.RunAsync(page);

            // page is closed implicitly
            try
            {
                browser.Disconnect();
            }
            catch { }
            var customresult = new WorkerResult {
                html = result, suburb = query, totalpages = totalpages, page = pagenumber
            };

            return(customresult);
        }