public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args) { var img = await http.GetAsync <JsonObject>($"screenshot?site={asset}"); observer?.Notify("WebpageScreenshot", "got screnshot", null); return(null != img ? img.screenshot?.data : null); }
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args) { Result result = await http.GetAsync <Result>($"getHtmlSource?site={asset}"); observer?.Notify("WebPageHthm", "got Html Source", null); return(result?.html); }
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args) { var proxy = new RestClient(http.Host); List <Result> result = await proxy.GetAsync <List <Result> >("getDollars"); observer?.Notify("Dollars", "got dollar values", null); return(result); }
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args) { if (String.IsNullOrWhiteSpace(asset) || IsLinkFile(asset)) { return(new List <string>()); } // asset = CleanURL(asset); HashSet <string> temp = await http.GetAsync <HashSet <string> >($"getAllLinks?site={asset}"); if (null == temp) { return(new List <string>()); } HashSet <string> temp2 = new HashSet <string>(); foreach (var link in temp) { if (!String.IsNullOrEmpty(link)) { if (_badChars.Contains(link[link.Length - 1])) { string fixedLink = link.Remove(link.Length - 1); temp2.Add(fixedLink); } else { temp2.Add(link); } } } IEnumerable <string> links = temp2; if (null != links && links.Any()) { var yuri = new Uri(asset); string host = yuri.Scheme + "://" + yuri.Host; links = links.Where(link => link.StartsWith(host) && !IsLinkFile(link));//sadece host adıyla başlayanları al. if (links.Any()) { IEnumerable <string> previousResults = await repository.GetResults(true, true); HashSet <string> doNotIncludeThose = new HashSet <string>(); foreach (string result in previousResults) { try { var list = JsonConvert.DeserializeObject <List <string> >(result); list.ForEach(i => doNotIncludeThose.Add(i)); } catch { } } links = links.Where(l => !doNotIncludeThose.Contains(l)); _ = repository.AddChildScan(asset, links);//clild' lari ekle asekron olarak. observer.Notify(nameof(Worker) + "_" + nameof(Run), "The Spider is working now.", links); } } return(links); }