protected override void ProcessRecord()
{
try
{
if (ParameterSetName == DefaultParameterSet)
{
SqlQuery = new SqlQuery(Sql, CommandTimeout, CUD, Parameters, Scalar, StoredProcedure, -1, Callback);
}
QueryNumber++;
ProgressRecord.CurrentOperation = "Running query number " + QueryNumber.ToString();
WriteProgress(ProgressRecord);
WriteVerbose("Running query number " + QueryNumber.ToString());
WriteVerbose("Running the following query: " + SqlQuery.Sql);
if (SqlQuery.Scalar)
{
RunScalarQuery();
}
else if (SqlQuery.CUD)
{
RunNonQuery();
}
else
{
RunQuery();
}
}
catch
{
CloseTransaction(true);
StopProcessing();
throw;
}
}
/// <summary>
/// Proceeds search in Google
/// </summary>
public override void DoSearch()
{
int page = 0;
HtmlDocument html = new HtmlDocument();
if (QueryRegex == null)
{
throw new Exception("Query is not set!");
}
while (true)
{
this.Driver.Url = this.queryType switch
{
QueryType.Number => string.Format(Url, QueryNumber.ToString(), page),
QueryType.String => string.Format(Url, Regex.Replace(QueryString, @"\s+", "+"), page),
_ => string.Format(Url, "Google", page),
};
// Wait until page is fully loaded
if (!Driver.WaitUntilPresent(By.CssSelector(".gb_ta.gb_he.gb_5a.gb_Pc"), 3))
{
break;
}
this.Driver.Navigate();
html.LoadHtml(this.Driver.PageSource);
var r = html.DocumentNode.SelectNodes("//div[@class='r']");
var st = html.DocumentNode.SelectNodes("//span[@class='st']");
if (r == null || r.Count != st.Count)
{
break;
}
for (int i = 0; i < r.Count; i++)
{
var link = r[i].SelectSingleNode(".//a").Attributes["href"].Value;
var title = r[i].SelectSingleNode(".//h3").InnerText;
var description = st[i].InnerText;
if (this.FindDatabase(title, description))
{
this.Results.Add(new Result(link, title, description, QueryNumber.ToString()));
}
}
page += r.Count;
}
}
/// <summary>
/// Proceeds search in Mail.Ru
/// </summary>
public override void DoSearch()
{
int page = 0;
HtmlDocument html = new HtmlDocument();
if (QueryRegex == null)
{
throw new Exception("Query is not set!");
}
while (true)
{
this.Driver.Url = this.queryType switch
{
QueryType.Number => string.Format(Url, QueryNumber.ToString(), page),
QueryType.String => string.Format(Url, Regex.Replace(QueryString, @"\s+", "+"), page),
_ => string.Format(Url, "Google", page),
};
this.Driver.Navigate();
html.LoadHtml(this.Driver.PageSource);
var li = html.DocumentNode.SelectNodes("//li[@class='result__li']");
if (li == null)
{
break;
}
else
{
page += li.Count;
}
for (int i = 0; i < li.Count; i++)
{
var link = li[i].SelectSingleNode(".//a").Attributes["href"].Value;
var urlText = li[i].SelectSingleNode(".//h3[@class='result__title']");
var fullText = li[i].SelectSingleNode(".//div[@class='SnippetResult-result']");
if (urlText != null && fullText != null)
{
string title = urlText.InnerText;
string description = fullText.InnerText;
if (this.FindDatabase(title, description))
{
this.Results.Add(new Result(link, title, description, QueryNumber.ToString()));
}
}
}
}
}
private FilterDefinition <TEntity> GetFilter <TEntity>(QueryNumber queryNumber)
{
switch (queryNumber.Operand)
{
case QueryNumberOperands.LessThen:
return(Builders <TEntity> .Filter.Lt(queryNumber.Field, queryNumber.Value));
case QueryNumberOperands.LessThenOrEquals:
return(Builders <TEntity> .Filter.Lte(queryNumber.Field, queryNumber.Value));
case QueryNumberOperands.Equals:
return(Builders <TEntity> .Filter.Eq(queryNumber.Field, queryNumber.Value));
case QueryNumberOperands.GreaterThen:
return(Builders <TEntity> .Filter.Gt(queryNumber.Field, queryNumber.Value));
case QueryNumberOperands.GreaterThenOrEquals:
return(Builders <TEntity> .Filter.Gte(queryNumber.Field, queryNumber.Value));
default:
return(null);
}
}
/// <summary>
/// Proceeds search in Yandex
/// </summary>
public override void DoSearch()
{
int page = 0;
HtmlDocument html = new HtmlDocument();
if (QueryRegex == null)
{
throw new Exception("Query is not set!");
}
while (true)
{
this.Driver.Url = this.queryType switch
{
QueryType.Number => string.Format(Url, QueryNumber.ToString(), page),
QueryType.String => string.Format(Url, Regex.Replace(QueryString, @"\s+", "+"), page),
_ => string.Format(Url, "Google", page),
};
// Wait until page is fully loaded
if (!Driver.WaitUntilPresent(By.CssSelector(".main.serp.i-bem.main_js_inited.serp_js_inited"), 3))
{
break;
}
html.LoadHtml(this.Driver.PageSource);
var serp = html.DocumentNode.SelectNodes("//li[@class='serp-item']");
// Get current page index
var onCurrentPage = html.DocumentNode.SelectSingleNode("//span[@class='pager__item pager__item_current_yes pager__item_kind_page']");
if (onCurrentPage == null || serp == null)
{
break;
}
if (int.Parse(onCurrentPage.InnerText) != page + 1)
{
break;
}
else
{
page++;
}
for (int i = 0; i < serp.Count; i++)
{
if (serp[i].Attributes["data-cid"] == null)
{
continue;
}
var link = serp[i].SelectSingleNode(".//a").Attributes["href"].Value;
var urlText = serp[i].SelectSingleNode(".//div[@class='organic__url-text']");
var fullText = serp[i].SelectSingleNode(".//span[@class='extended-text__full']");
if (urlText != null && fullText != null)
{
string title = urlText.InnerText;
string description = fullText.InnerText;
if (this.FindDatabase(title, description))
{
this.Results.Add(new Result(link, title, description, QueryNumber.ToString()));
}
}
}
}
}
/// <summary>
/// Proceeds search in DuckDuckGo (HTML version)
/// </summary>
public override void DoSearch()
{
HtmlDocument html = new HtmlDocument();
if (QueryRegex == null)
{
throw new Exception("Query is not set!");
}
this.Driver.Url = this.queryType switch
{
QueryType.Number => string.Format(Url, QueryNumber.ToString()),
QueryType.String => string.Format(Url, Regex.Replace(QueryString, @"\s+", "+")),
_ => string.Format(Url, "Google"),
};
while (true)
{
html.LoadHtml(this.Driver.PageSource);
HtmlNodeCollection results = html.DocumentNode.SelectNodes(".//div[@class='links_main links_deep result__body']");
if (results == null)
{
break;
}
for (int i = 0; i < results.Count; i++)
{
var anchor = results[i].SelectSingleNode(".//a[@class='result__a']");
var snippet = results[i].SelectSingleNode(".//a[@class='result__snippet']");
if (anchor != null && snippet != null)
{
string link = anchor.Attributes["href"].Value;
string title = anchor.InnerText;
string description = snippet.InnerText;
if (this.FindDatabase(title, description, false))
{
this.Results.Add(new Result(link, title, description, QueryNumber.ToString()));
}
}
}
// Check presence of 'Next' button...
var by = By.XPath("//*[@class='btn btn--alt' and @value='Next']");
if (Driver.FindElements(by).Count == 0)
{
break;
}
// ... and click on it!
var next = Driver.FindElement(by);
if (next != null)
{
next.Click();
}
else
{
break;
}
}
}
/// <summary>
/// Proceeds search in Bing
/// </summary>
public override void DoSearch()
{
int page = 1;
int total = 0;
HtmlDocument html = new HtmlDocument();
if (QueryRegex == null)
{
throw new Exception("Query is not set!");
}
while (true)
{
this.Driver.Url = this.queryType switch
{
QueryType.Number => string.Format(Url, QueryNumber.ToString(), page),
QueryType.String => string.Format(Url, Regex.Replace(QueryString, @"\s+", "+"), page),
_ => string.Format(Url, "Google", page),
};
// Wait until page is fully loaded
if (!Driver.WaitUntilPresent(By.Id("b_tween"), 3))
{
break;
}
html.LoadHtml(this.Driver.PageSource);
HtmlNodeCollection algo = html.DocumentNode.SelectNodes(".//li[@class='b_algo']");
// Gets total count of results for 'first' value in url
if (total == 0)
{
HtmlNode count = html.DocumentNode.SelectSingleNode(".//span[@class='sb_count']");
int[] digits = Number.GetDigits(count.InnerText).ToArray();
total = digits[digits.Length - 1];
}
if (algo == null || page > total)
{
break;
}
else
{
page += algo.Count + 1;
}
for (int i = 0; i < algo.Count; i++)
{
var link = algo[i].SelectSingleNode(".//a").Attributes["href"].Value;
var urlText = algo[i].SelectSingleNode(".//a");
var fullText = algo[i].SelectSingleNode(".//p");
if (urlText != null && fullText != null)
{
string title = urlText.InnerText;
string description = fullText.InnerText;
if (this.FindDatabase(title, description))
{
this.Results.Add(new Result(link, title, description, QueryNumber.ToString()));
}
}
}
}
}
