public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
var img = await http.GetAsync <JsonObject>($"screenshot?site={asset}");
observer?.Notify("WebpageScreenshot", "got screnshot", null);
return(null != img ? img.screenshot?.data : null);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
Result result = await http.GetAsync <Result>($"getHtmlSource?site={asset}");
observer?.Notify("WebPageHthm", "got Html Source", null);
return(result?.html);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
var proxy = new RestClient(http.Host);
List <Result> result = await proxy.GetAsync <List <Result> >("getDollars");
observer?.Notify("Dollars", "got dollar values", null);
return(result);
}
public override async Task <object> Run_Internal(ProxyObserver observer, RestClient http, string asset, IScanRepository repository, object args)
{
if (String.IsNullOrWhiteSpace(asset) || IsLinkFile(asset))
{
return(new List <string>());
}
// asset = CleanURL(asset);
HashSet <string> temp = await http.GetAsync <HashSet <string> >($"getAllLinks?site={asset}");
if (null == temp)
{
return(new List <string>());
}
HashSet <string> temp2 = new HashSet <string>();
foreach (var link in temp)
{
if (!String.IsNullOrEmpty(link))
{
if (_badChars.Contains(link[link.Length - 1]))
{
string fixedLink = link.Remove(link.Length - 1);
temp2.Add(fixedLink);
}
else
{
temp2.Add(link);
}
}
}
IEnumerable <string> links = temp2;
if (null != links && links.Any())
{
var yuri = new Uri(asset);
string host = yuri.Scheme + "://" + yuri.Host;
links = links.Where(link => link.StartsWith(host) && !IsLinkFile(link));//sadece host adıyla başlayanları al.
if (links.Any())
{
IEnumerable <string> previousResults = await repository.GetResults(true, true);
HashSet <string> doNotIncludeThose = new HashSet <string>();
foreach (string result in previousResults)
{
try
{
var list = JsonConvert.DeserializeObject <List <string> >(result);
list.ForEach(i => doNotIncludeThose.Add(i));
}
catch { }
}
links = links.Where(l => !doNotIncludeThose.Contains(l));
_ = repository.AddChildScan(asset, links);//clild' lari ekle asekron olarak.
observer.Notify(nameof(Worker) + "_" + nameof(Run), "The Spider is working now.", links);
}
}
return(links);
}