public void ImgsHelper(ScraperEngineResponse response)
{
//var results = response.Doc.DocumentNode.SelectNodes("//img[@alt='']");
var results = response.Doc.DocumentNode.SelectNodes("//img[not(@alt)] | //img[@alt='']");
if (results != null)
{
List <Task> tasks = new List <Task>();
tasks.Add(Task.Run(() =>
{
foreach (var result in results)
{
if (result.OuterHtml == null)
{
continue;
}
if (!ImgsWithNoAlt.ContainsKey(result.OuterHtml))
{
ImgsWithNoAlt.Add(result.OuterHtml, response.Url);
}
}
StateHasChangedDelegate?.Invoke();
}));
}
}
public async Task ValidateOTP()
{
var cancellationToken = new CancellationTokenSource().Token;
await coWINAuthService.ValidateMobileOTPAsync(OTP, cancellationToken);
var beneficiaryDTOList = await coWINAppointmentService.FetchBeneficiaries(cancellationToken);
if (beneficiaryDTOList != null)
{
foreach (var beneficiaryDTO in beneficiaryDTOList)
{
if (beneficiaryDTO != null)
{
var beneficiary = new Models.Beneficiary();
beneficiary.Name = beneficiaryDTO.Name;
beneficiary.PhotoId = beneficiaryDTO.PhotoIdType;
beneficiary.IdNumber = beneficiaryDTO.PhotoIdNumber;
Beneficiaries.Add(beneficiary);
}
}
}
HideLoginDetails = true;
HideLogoutDetails = false;
HideSearchCriteria = false;
StateHasChangedDelegate?.Invoke();
}
public async Task StartSearchAsync()
{
var cancellationToken = new CancellationTokenSource().Token;
string date = DateTime.Now.ToString("dd-MM-yyyy");
var sessionsResponse = await coWINAppointmentService.FetchSessionsByDistrictIdAndDate(
DistrictsDTO.Where(x => x.DistrictName == District).FirstOrDefault().DistrictId.ToString(), date, cancellationToken);
foreach (var center in sessionsResponse.Centers)
{
if (center != null)
{
foreach (var session in center.Sessions)
{
if (session != null)
{
var vaccineCenter = new VaccineCenter();
vaccineCenter.CenterName = center.Name;
vaccineCenter.Availability = session.AvailableCapacity;
vaccineCenter.AvailabilityDose1 = session.AvailableCapacityDose1;
vaccineCenter.AvailabilityDose2 = session.AvailableCapacityDose2;
vaccineCenter.MinimumAgeLimit = session.MinAgeLimit;
vaccineCenter.VaccineType = session.Vaccine;
AvailableVaccineCenters.Add(vaccineCenter);
}
}
}
}
HideSearchResults = false;
StateHasChangedDelegate?.Invoke();
}
public void LogOut()
{
coWINUtilities.TokenText = null;
HideLoginDetails = false;
HideLogoutDetails = true;
HideSearchCriteria = true;
HideSearchResults = true;
StateHasChangedDelegate?.Invoke();
}
public async Task GetOTP()
{
var cancellationToken = new CancellationTokenSource().Token;
await coWINAuthService.GenerateMobileOTPAsync(MobileNumber, cancellationToken);
DisableOTPField = false;
DisableOTPButton = false;
StateHasChangedDelegate?.Invoke();
}
public void DescriptionsHelper(ScraperEngineResponse response)
{
var results = response.Doc.DocumentNode.SelectNodes("//meta[@name='description']");
if (results != null)
{
foreach (var result in results)
{
var innerText = result.GetAttributeValue("content", "none");
if (innerText.Equals("none") || innerText.Equals(""))
{
TitleDescCheck description =
new TitleDescCheck("Missing description", innerText, response.Url);
EmptyDescriptions.Add(description);
AllDescriptions.Add(description);
}
else if (innerText.Length > 160)
{
TitleDescCheck description = new TitleDescCheck("Too long", innerText, response.Url);
LongDescriptions.Add(description);
AllDescriptions.Add(description);
}
else if (innerText.Length <= 50)
{
TitleDescCheck description = new TitleDescCheck("Too short", innerText, response.Url);
ShortDescriptions.Add(description);
AllDescriptions.Add(description);
}
else
{
TitleDescCheck description = new TitleDescCheck("Good", innerText, response.Url);
HealthyDescriptions.Add(description);
AllDescriptions.Add(description);
}
StateHasChangedDelegate?.Invoke();
}
}
}
public void TitlesHelper(ScraperEngineResponse response)
{
var results = response.Doc.DocumentNode.SelectNodes("//title");
if (results != null)
{
foreach (var result in results)
{
if (result.OuterHtml == null || result.InnerText == "")
{
TitleDescCheck title = new TitleDescCheck("Missing title", result.InnerText, response.Url);
EmptyTitles.Add(title);
AllTitles.Add(title);
}
else if (result.InnerText.Length >= 60)
{
TitleDescCheck title = new TitleDescCheck("Too long", result.InnerText, response.Url);
LongTitles.Add(title);
AllTitles.Add(title);
}
else if (result.InnerText.Length <= 40)
{
TitleDescCheck title = new TitleDescCheck("Too short", result.InnerText, response.Url);
ShortTitles.Add(title);
AllTitles.Add(title);
}
else
{
TitleDescCheck title = new TitleDescCheck("Good", result.InnerText, response.Url);
HealthyTitles.Add(title);
AllTitles.Add(title);
}
StateHasChangedDelegate?.Invoke();
}
}
}
protected async void StateHasChanged()
{
StateHasChangedDelegate?.Invoke();
await dbContext.SaveChangesAsync();
}
public async Task <string> AnalyzeLinks(string websiteName)
{
AllLinks = new List <LinkCheck>();
HealthyLinks = new List <LinkCheck>();
BrokenLinks = new List <LinkCheck>();
BlockedLinks = new List <LinkCheck>();
RedirectLinks = new List <LinkCheck>();
LinksWithIssues = new List <LinkCheck>();
Helper.SetUrl(websiteName);
await foreach (var response in ResponsesHref)
{
var url = FormatHrefForAnalyzer(response.Url);
//href includes tel and mailto, we don't want these as our links
if (url.Contains("tel:") || url.Contains("mailto:") || url.Contains("callto:"))
{
continue;
}
List <Task> tasks = new List <Task>();
tasks.Add(Task.Run(() =>
{
try
{
_doc = _web.Load(url);
}
catch (Exception e)
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Has Issues", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
LinksWithIssues.Add(result);
}
return;
}
switch (_web.StatusCode)
{
case HttpStatusCode.NoContent:
case HttpStatusCode.IMUsed:
case HttpStatusCode.Accepted:
case HttpStatusCode.OK:
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Healthy", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
HealthyLinks.Add(result);
}
break;
}
case HttpStatusCode.Unauthorized:
case HttpStatusCode.FailedDependency:
case HttpStatusCode.MethodNotAllowed:
case HttpStatusCode.ExpectationFailed:
case HttpStatusCode.NotAcceptable:
case HttpStatusCode.Forbidden:
case HttpStatusCode.NetworkAuthenticationRequired:
case HttpStatusCode.TooManyRequests:
case HttpStatusCode.UnavailableForLegalReasons:
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Blocked", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
BlockedLinks.Add(result);
}
break;
}
case HttpStatusCode.Gone:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.MisdirectedRequest:
case HttpStatusCode.NotImplemented:
case HttpStatusCode.ServiceUnavailable:
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Has Issues", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
LinksWithIssues.Add(result);
}
break;
}
case HttpStatusCode.Ambiguous:
case HttpStatusCode.Redirect:
case HttpStatusCode.Moved:
case HttpStatusCode.PermanentRedirect:
case HttpStatusCode.TemporaryRedirect:
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Redirect", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
RedirectLinks.Add(result);
}
break;
}
default:
{
if (!Results.Contains(url))
{
Results.Add(url);
LinkCheck result = new LinkCheck("Broken", response.Url, url, _web.StatusCode);
AllLinks.Add(result);
BrokenLinks.Add(result);
}
break;
}
}
StateHasChangedDelegate?.Invoke();
}));
}
return("done");
}
public async Task BfsAsync(int maxWidth)
{
Dictionary <string, bool> visited = new Dictionary <string, bool>();
Queue <HtmlDocument> sites = new Queue <HtmlDocument>();
AllUrls = new List <string>();
int width = 0;
var root = _web.Load(BaseDomain);
visited["/"] = true;
sites.Enqueue(root);
AllUrls.Add(BaseDomain);
while (sites.Count != 0)
{
var url = sites.Dequeue();
if (url == null)
{
continue;
}
//System.Diagnostics.Debug.Print("Root loaded: " + _web.ResponseUri.AbsoluteUri);
var neighborUrls = url.DocumentNode.SelectNodes("//a[@href]");
if (neighborUrls == null)
{
continue;
}
int visitedNeighborsCount = 0;
while (width < maxWidth && visitedNeighborsCount < neighborUrls.Count)
{
List <Task> tasks = new List <Task>();
for (int i = visitedNeighborsCount; i < neighborUrls.Count; i++, visitedNeighborsCount++)
{
HtmlNode neighbor = neighborUrls[i];
if (width++ >= maxWidth)
{
break;
}
string href = neighbor.GetAttributeValue("href", string.Empty);
if (href == string.Empty || visited.ContainsKey(href))
{
continue;
}
visited[href] = true;
tasks.Add(Task.Run(() =>
{
string neighborUrl = FormatHref(href);
System.Diagnostics.Debug.Print(neighborUrl);
if (neighborUrl != "invalid" && neighborUrl != "robots.txt disallowed")
{
//System.Diagnostics.Debug.Print("Loading: " + neighborUrl);
HtmlDocument doc = new HtmlDocument();
try { doc = _web.Load(neighborUrl); }
catch { return; }
if (_web.StatusCode != HttpStatusCode.OK)
{
return;
}
//System.Diagnostics.Debug.Print("Loaded: " + neighborUrl);
AllUrls.Add(neighborUrl);
StateHasChangedDelegate?.Invoke();
sites.Enqueue(doc);
}
}));
}
await Task.WhenAll(tasks);
width = AllUrls.Count;
}
}
}
public async Task <List <SerpPost> > TrackSomething(string keyword, string userAgent, string location, string enteredpage, string username)
{
Organic.Clear();
OrganicFound.Clear();
Reklame.Clear();
Position = 0;
var responses = Engine.GetGooglePages(keyword, userAgent, location, 4, 5000);
await foreach (var response in responses)
{
GiveAds(response);
var resultsOnly = response.Doc.GetElementbyId("rso").SelectNodes(".//div[@class='g']");//kw, position, datetime, url
if (resultsOnly == null)
{
return(null);
}
foreach (var result in resultsOnly)
{
Position++;
var divClass = result.GetAttributeValue("class", "");
if (divClass != "g")
{
continue;
}
var a = result.SelectSingleNode(".//a");
var url = a.Attributes["href"].Value;
var title = a.SelectSingleNode(".//h3").InnerText;
Organic.Add(new Result(url, title, Position));
StateHasChangedDelegate?.Invoke();
if (enteredpage != null) //Checks if entered URL is on Google search
{
if (!enteredpage.StartsWith("https://")) //if http || wwww
{
if (!enteredpage.StartsWith("http://")) //uri needs http
{
enteredpage = "http://" + enteredpage;
}
Uri uri = new Uri(enteredpage);
var temp = uri.Host;
if (temp.Contains("www."))
{
temp = temp.Substring(4);
}
if (url.Contains(temp))
{
Found = true;
string date = DateTime.Now.ToString("yyyy-MM-dd");
Organic[Organic.Count - 1].Found = true;
OrganicFound.Add(new SerpPost(date, keyword, enteredpage, Position, username));
}
}
else //if https
{
enteredpage = enteredpage.Substring(8); //remove https part bcs uri needs http
enteredpage = "http://" + enteredpage;
Uri uri = new Uri(enteredpage);
var temp = uri.Host;
if (temp.Contains("www."))
{
temp = temp.Substring(4);
}
if (url.Contains(temp))
{
Found = true;
string date = DateTime.Now.ToString("yyyy-MM-dd");
Organic[Organic.Count - 1].Found = true;
OrganicFound.Add(new SerpPost(date, keyword, enteredpage, Position, username));
}
}
}
}
}
if (OrganicFound.Count == 0)
{
string date = DateTime.Now.ToString("yyyy-MM-dd");
OrganicFound.Add(new SerpPost(date, keyword, enteredpage, 101, username));
}
return(OrganicFound);
}
private void GiveAds(ScraperEngineResponse response)
{
var doorstep = response.Doc.GetElementbyId("tads");
if (doorstep != null)
{
HtmlNodeCollection topads;
try
{
topads = response.Doc.GetElementbyId("tads").SelectNodes("//li[@class='ads-ad']");
if (topads == null)
{
return;
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
foreach (var adresult in topads)
{
var divClass = adresult.GetAttributeValue("class", "");
if (divClass != "ads-ad")
{
continue;
}
var url = adresult.SelectSingleNode(".//cite[@class='UdQCqe']").InnerText;
var domain = url;
var filtrirana = "";
var filtereddomain = "";
if (domain != null)
{
if (!domain.StartsWith("https://")) //if domain http || www
{
if (!domain.StartsWith("http://")) //uri needs http
{
domain = "http://" + domain;
}
Uri uri = new Uri(domain);
filtereddomain = uri.Host; //e.g. http://www.example.com
string[] subdomain = filtereddomain.Split(new char[] { '.' }); //split string
if (filtereddomain.Contains("www."))
{
var count = subdomain[0].Length; //remove www
filtrirana = filtereddomain.Remove(0, count + 1);
}
else
{
var count = subdomain[0].Length;
filtrirana = filtereddomain.Remove(0, count + 1);
}
}
else //if domain https
{
domain = domain.Substring(8);
domain = "http://" + domain;
Uri uri = new Uri(domain);
filtereddomain = uri.Host;
if (filtereddomain.Contains("www."))
{
filtereddomain = filtereddomain.Substring(4);
}
Console.WriteLine(filtereddomain);
}
}
if (!filtereddomain.StartsWith("https://"))
{
filtereddomain = "https://" + filtereddomain;
}
var subject = adresult.SelectSingleNode(".//h3").InnerText;
WebUtility.HtmlDecode(subject);
var desc = adresult.SelectSingleNode(".//div[@class='ads-creative']").InnerText;
Advert tempres = new Advert(filtereddomain, filtrirana, subject, desc);
Reklame.Add(tempres);
StateHasChangedDelegate?.Invoke();
}
}
}
