static void Main(string[] args)
{
string googleKey = "yWje5FlQFHIDjiuRQzVyhVmvn9dSjsPp";
if (args.Length == 0 || (args.Length == 1 && (args[0] == "?" || args[0] == "/?" || args[0] == "-?" || args[0].ToLower() == "help")))
{
Console.WriteLine("Usage: google.exe <query>\n");
return;
}
string query = args[0];
// Create a Google SOAP client proxy, generated by:
// c:\> wsdl.exe http://api.google.com/GoogleSearch.wsdl
GoogleSearchService googleSearch = new GoogleSearchService();
GoogleSearchResult results = googleSearch.doGoogleSearch(googleKey, query, 0, 10, false, "", false, "", "latin1", "latin1");
if (results.resultElements != null)
{
foreach (ResultElement result in results.resultElements)
{
Console.WriteLine();
Console.WriteLine(result.title.Replace("<b>", "").Replace("</b>", ""));
Console.WriteLine(result.URL);
//Console.WriteLine(result.snippet);
Console.WriteLine();
}
}
}
/// <summary>
/// Perform a Google search and return the results. Only return
/// 100 results.
/// </summary>
/// <param name="query">What to search for.</param>
/// <returns>The URL's that google returned for the search.</returns>
public static List <ResultElement> GetResults(String query)
{
List <ResultElement> result = new List <ResultElement>();
int resultCount = 0;
int top = 0;
do
{
GoogleSearchResult r = search.doGoogleSearch(key,
query, top, 10, false, "", false, "", "latin1", "latin1");
resultCount = r.resultElements.GetLength(0);
if (r != null)
{
foreach (ResultElement element in r.resultElements)
{
result.Add(element);
}
}
top = top + 10;
Console.WriteLine("Searching Google: " + top);
} while ((resultCount >= 10) && (top < 100));
return(result);
}
/// <summary>
/// This is the main interface, which calls all of the
/// helper methods. We create the google objects, do the search, display the results inside
/// this code.
/// </summary>
private void GoogleHandler()
{
// Create a Google Search object
GoogleSearchService gss = new GoogleSearchService();
// Invoke the search method
GoogleSearchResult results = gss.doGoogleSearch(LICENSE_KEY, BuildQuery(),
0, 10, false, string.Empty, false, string.Empty, string.Empty, string.Empty);
// Extract the estimated number of results for the search and display it
lblResults.Text = "Results Found: " + results.estimatedTotalResultsCount;
lblResults.Visible = true;
DataTable table = new DataTable();
DataColumn col1 = new DataColumn("Results", typeof(string));
table.Columns.Add(col1);
int pointer = 0;
while (pointer < results.resultElements.Length)
{
DataRow row = table.NewRow();
row[0] = FormatResults(results.resultElements[pointer]);
table.Rows.Add(row);
pointer++;
}
dgResults.DataSource = table;
dgResults.DataBind();
}
public void SaveToDisk(GoogleSearchResult Results)
{
StreamWriter FWrite = null;
try
{
FWrite = new StreamWriter(FileToSave, false, System.Text.Encoding.ASCII);
foreach (ResultElement r in Results.resultElements)
{
FWrite.WriteLine(r.URL);
FWrite.WriteLine(r.title);
FWrite.WriteLine(r.snippet);
FWrite.WriteLine();
FWrite.WriteLine();
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
finally
{
if (FWrite != null)
{
FWrite.Close();
}
}
}
private void Search_Click(object sender, System.EventArgs e)
{
GoogleSearchService s = new GoogleSearchService();
/*
* s.doGoogleSearch(string license,string query, int start, int maxResults,
* bool filter,string restrict,bool safeSearch, string lr, string ie, string oe)
*/
try
{
int start = (Convert.ToInt32(TextBox2.Text) - 1) * 10;
GoogleSearchResult r = s.doGoogleSearch(licKey, txtSearchString.Text, start, maxResults, false, string.Empty, false, string.Empty, string.Empty, string.Empty);
// Extract the estimated number of results for the search and display it
int estResults = r.estimatedTotalResultsCount;
lblHits.Text = Convert.ToString(estResults) + " Results found";
DataSet ds1 = new DataSet();
DataSet ds = FillGoogleDS(ds1, r);
DataGrid1.DataSource = ds;
DataGrid1.DataBind();
}
catch (Exception ex)
{
lblHits.Text = ex.Message;
return;
}
return;
}
public void CallGoogleSearch(string toSearch)
{
GoogleSearchService Search = null;
GoogleSearchResult Results = null;
try
{
Search = new GoogleSearchService();
Results = Search.doGoogleSearch(
"Vaš Licence Key",
toSearch, 0, 10, false, "", false, "", "", "");
}
catch (System.Net.WebException e)
{
Console.WriteLine(e.Message);
return;
}
Process IE = new Process();
IE.StartInfo.FileName = "iexplore.exe";
IE.StartInfo.Arguments = Results.resultElements[0].URL;
IE.Start();
SaveToDisk(Results);
}
/// <summary>
/// For the given URL check how many links the URL has.
/// </summary>
/// <param name="url">The URL to check.</param>
/// <returns>The number of links that URL has.</returns>
public static int getLinkCount(String url)
{
int result = 0;
String query = "link:" + url;
GoogleSearchResult r = search.doGoogleSearch(key,
query, 0, 10, false, "", false, "", "latin1", "latin1");
result = r.estimatedTotalResultsCount;
return(result);
}
private void GetMovieUrlsCompleted(object sender, OpenReadCompletedEventArgs e)
{
_webClient.OpenReadCompleted -= GetMovieUrlsCompleted;
var userCallback = e.UserState as Action <List <GoogleSearchResult>, Exception>;
if (userCallback == null)
{
return;
}
if (e.Error != null)
{
userCallback(null, e.Error);
return;
}
var resultList = new List <GoogleSearchResult>();
XmlReader reader = JsonReaderWriterFactory.CreateJsonReader(e.Result, XmlDictionaryReaderQuotas.Max);
while (reader.Read())
{
//string xml = reader.ReadOuterXml();
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "cursor")
{
break;
}
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "item")
{
var resultItem = new GoogleSearchResult();
reader.ReadToDescendant("unescapedUrl");
resultItem.UnescapedUrl = reader.ReadElementContentAsString();
reader.MoveToContent();
resultItem.Url = reader.ReadElementContentAsString();
reader.MoveToContent();
resultItem.VisibleUrl = reader.ReadElementContentAsString();
reader.ReadToFollowing("titleNoFormatting");
reader.MoveToContent();
resultItem.TitleNoFormatting = reader.ReadElementContentAsString();
reader.MoveToContent();
resultItem.Content = reader.ReadElementContentAsString();
var match = new Regex(@"/title/tt\d+/$");
if (match.IsMatch(resultItem.Url))
{
resultList.Add(resultItem);
}
}
}
userCallback(resultList, null);
}
/// <summary>
/// Search button: do a search, display number of results
/// </summary>
private void searchButton_Click(object sender, System.EventArgs e)
{
// Create a Google Search object
GoogleSearchService s = new GoogleSearchService();
// Invoke the search method
GoogleSearchResult r = s.doGoogleSearch(keyBox.Text, searchBox.Text,
0, 1, false, "", false, "", "", "");
// Extract the estimated number of results for the search and display it
int estResults = r.estimatedTotalResultsCount;
searchResultLabel.Text = Convert.ToString(estResults);
}
/// <summary>
/// Handles the Click event of the Search control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="T:System.EventArgs"/> instance containing the event data.</param>
private void Search_Click(object sender, EventArgs e)
{
string searchText = txtSearchString.Text;
GoogleSearchService s = new GoogleSearchService();
/*
* s.doGoogleSearch(string license,string query, int start, int maxResults,
* bool filter,string restrict,bool safeSearch, string lr, string ie, string oe)
*/
try
{
//Overrides following
if (restrictToThisDomain.Length != 0 && searchText.IndexOf("site:") == -1)
{
searchText = searchText += " site:" + restrictToThisDomain.Trim();
}
if (searchThisSiteOnly && searchText.IndexOf("site:") == -1)
{
searchText = searchText += " site:" + (Request.Url.Host);
}
//debug only
//txtSearchString.Text = searchText;
int start = (Convert.ToInt32(TextBox2.Text) - 1) * 10;
GoogleSearchResult r = s.doGoogleSearch(licKey, searchText, start, maxResults, false, string.Empty, false, string.Empty, string.Empty, string.Empty);
// Extract the estimated number of results for the search and display it
int estResults = r.estimatedTotalResultsCount;
lblHits.Text = Convert.ToString(estResults) + " Results found";
DataSet ds1 = new DataSet();
DataSet ds = FillGoogleDS(ds1, r);
DataGrid1.DataSource = ds;
DataGrid1.DataBind();
}
catch (Exception ex)
{
lblHits.Text = ex.Message;
return;
}
return;
}
public int SearchGoogle(string word)
{
int total = 0;
string googleUrl = ConfigurationManager.AppSettings[Constants.GOOGLE_SEARCH_URL];
string googleSearchEngine = ConfigurationManager.AppSettings[Constants.GOOGLE_SEARCH_ENGINE];
string googleApiKey = ConfigurationManager.AppSettings[Constants.GOOGLE_SEARCH_KEY];
string fullUrl = string.Format(googleUrl, googleApiKey, googleSearchEngine, word);
using (var httpWebResponse = (HttpWebResponse)WebRequest.Create(fullUrl).GetResponse())
{
GoogleSearchResult searchResult = DeserializedObject <GoogleSearchResult>(httpWebResponse);
total = searchResult.SearchInformation.TotalResults != 0 ? searchResult.SearchInformation.TotalResults : total;
}
return(total);
}
public void DoQuery(Query query,
IQueryResult result,
IQueryableChangeData changeData)
{
StringBuilder sb = new StringBuilder();
foreach (QueryPart part in query.Parts)
{
if (part is QueryPart_Human)
{
sb.Append(((QueryPart_Human)part).QueryString);
}
else if (part is QueryPart_Text)
{
sb.Append(((QueryPart_Text)part).Text);
}
else
{
continue; // ignore other query parts
}
}
Logger.Log.Debug("Querying google for '" + sb.ToString() + "'");
GoogleSearchResult gsr = gss.doGoogleSearch(googleKey,
sb.ToString(),
0, maxResults,
false, "", false, "", "", "");
ArrayList hits = new ArrayList();
int rank = 0;
foreach (ResultElement elt in gsr.resultElements)
{
Hit hit = FromGoogleResultElement(elt, rank);
hits.Add(hit);
++rank;
}
result.Add(hits, gsr.estimatedTotalResultsCount);
}
/// <summary>
/// Fills the google DS.
/// </summary>
/// <param name="ds">The ds.</param>
/// <param name="srchResult">The SRCH result.</param>
/// <returns></returns>
private DataSet FillGoogleDS(DataSet ds, GoogleSearchResult srchResult)
{
try
{
ds = CreateGoogleDS(ds);
int i = 0;
DataRow dr;
string strURL = null;
for (i = 0; i < srchResult.resultElements.Length; i++)
{
dr = ds.Tables["GoogleSearch"].NewRow();
strURL = srchResult.resultElements[i].URL.ToString();
dr["Title"] = "<a href=" + strURL + " Target='" + Target + "' >" + srchResult.resultElements[i].title.ToString() + "</a>";
if (showSnippet)
{
dr["Snippet"] = srchResult.resultElements[i].snippet.ToString();
}
if (showSummary)
{
dr["Summary"] = srchResult.resultElements[i].summary.ToString();
}
if (showURL)
{
dr["URL"] = "<a href=" + strURL + " Target='" + Target + "' >" + strURL + "</a>";
}
ds.Tables["GoogleSearch"].Rows.Add(dr);
}
}
catch (Exception e)
{
lblHits.Text = e.Message;
return(null);
}
return(ds);
}
public void GoogleSearchResult_Should_be_Parsed()
{
var rules = new Dictionary <Func <dynamic, object>, object>()
{
{ x => x.Kind, "customsearch#search" },
// *********** Url ****************************
{ x => x.Url.Type, "application/json" },
{ x => x.Url.Template, "https://www.googleapis.com/customsearch/v1?q={searchTerms}&num={count?}&start={startIndex?}&lr={language?}&safe={safe?}&cx={cx?}&cref={cref?}&sort={sort?}&filter={filter?}&gl={gl?}&cr={cr?}&googlehost={googleHost?}&c2coff={disableCnTwTranslation?}&hq={hq?}&hl={hl?}&nsc={nsc?}&siteSearch={siteSearch?}&siteSearchFilter={siteSearchFilter?}&exactTerms={exactTerms?}&excludeTerms={excludeTerms?}&linkSite={linkSite?}&orTerms={orTerms?}&relatedSite={relatedSite?}&dateRestrict={dateRestrict?}&lowRange={lowRange?}&highRange={highRange?}&searchType={searchType}&fileType={fileType?}&rights={rights?}&imgSize={imgSize?}&imgType={imgType?}&imgColorType={imgColorType?}&imgDominantColor={imgDominantColor?}&alt=json" },
// ********** Queries ****************************
{ x => x.Queries.NextPage[0].Title, "Google Custom Search - flowers" },
{ x => x.Queries.NextPage[0].TotalResults, 10300000L },
{ x => x.Queries.NextPage[0].SearchTerms, "flowers" },
{ x => x.Queries.NextPage[0].Count, 10 },
{ x => x.Queries.NextPage[0].StartIndex, 11 },
{ x => x.Queries.NextPage[0].InputEncoding, "utf8" },
{ x => x.Queries.NextPage[0].OutputEncoding, "utf8" },
{ x => x.Queries.NextPage[0].Cx, "013036536707430787589:_pqjad5hr1a" },
{ x => x.Queries.PreviousPage, null },
{ x => x.Queries.Request[0].Title, "Google Custom Search - flowers" },
{ x => x.Queries.Request[0].TotalResults, 10300000L },
{ x => x.Queries.Request[0].SearchTerms, "flowers" },
{ x => x.Queries.Request[0].Count, 10 },
{ x => x.Queries.Request[0].StartIndex, 1 },
{ x => x.Queries.Request[0].InputEncoding, "utf8" },
{ x => x.Queries.Request[0].OutputEncoding, "utf8" },
{ x => x.Queries.Request[0].Cx, "013036536707430787589:_pqjad5hr1a" },
// ************ Context *************************
{ x => x.Context.Title, "Custom Search" },
// ************ Items ***************************
{ x => x.Items[0].Kind, "customsearch#result" },
{ x => x.Items[0].Title, "Flower - Wikipedia, the free encyclopedia" },
{ x => x.Items[0].HtmlTitle, "<b>Flower</b> - Wikipedia, the free encyclopedia" },
{ x => x.Items[0].Link, "http://en.wikipedia.org/wiki/Flower" },
{ x => x.Items[0].DisplayLink, "en.wikipedia.org" },
{ x => x.Items[0].Snippet, "A flower, sometimes known as a bloom or blossom, is the reproductive structure found in flowering plants (plants of the division Magnoliophyta, ..." },
{ x => x.Items[0].HtmlSnippet, "A <b>flower</b>, sometimes known as a bloom or blossom, is the reproductive structure <br> found in flowering plants (plants of the division Magnoliophyta, <b>... </b>" },
};
var serializer = new DataContractJsonSerializer(typeof(GoogleSearchResult));
GoogleSearchResult actualObject = null;
using (var stream = File.OpenRead("GoogleSearchJson.txt")) {
stream.Seek(Encoding.UTF8.GetPreamble().Length, SeekOrigin.Begin); // HACK, skip Unicode preambule
actualObject = serializer.ReadObject(stream) as GoogleSearchResult;
}
Assert.IsNotNull(actualObject, "GoogleSearchResult cannot be deserialized. Result is null");
foreach (var rule in rules)
{
var actual = rule.Key(actualObject);
var expected = rule.Value;
Assert.AreEqual(expected, actual);
}
}
public StackoverflowDataSource(GoogleSearchResult i_Result)
{
m_ID = urlToStackoverflowId(i_Result.Url);
GoogleResultIndex = i_Result.Index;
}
public async Task <IActionResult> Get([FromQuery] ScrapeRequest scrapeRequest)
{
//input validation has already handled here
string keywords = scrapeRequest.Keywords;
string matchingKeywords = scrapeRequest.MatchingKeywords;
string baseUrl = "https://www.google.com.au/search";
string unknownErrorMessage = "Result Div not found. Please contact the dev team to update this software.";
try
{
//build final search url
string finalUrl = _scrapingService.BuildGoogleSearchUrl(baseUrl, keywords, scrapeRequest.ResultNum);
//open browser to get full page source
_webDriver.Navigate().GoToUrl(finalUrl);
string pageSource = _webDriver.PageSource;
_webDriver.Quit();
//finalUrl = "https://www.ebay.com";
//optional: download html to a file
await _scrapingService.SaveToFileAsync(pageSource, keywords, DateTime.Today);
logger.Here().Information("Saved page source to file successfully");
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(pageSource);
var rsoDiv = htmlDocument.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("id", "") == "rso").ToList();
if (rsoDiv.Count == 0)
{
//TODO: this should be 500 error
return(ErrorResponseBadRequest(logger.Here(), unknownErrorMessage, ErrorType.Unknown));
}
var searchResultComponentDivs = rsoDiv[0].Descendants("div")
.Where(node => node.GetAttributeValue("class", "") == "bkWMgd")
.Where(node => !node.Descendants("div").Any(childNode => childNode.GetAttributeValue("class", "") == "g kno-kp mnr-c g-blk"))
.ToList();
if (searchResultComponentDivs.Count == 0)
{
//TODO: this should be 500 error
return(ErrorResponseBadRequest(logger.Here(), unknownErrorMessage, ErrorType.Unknown));
}
var searchResultDivs = new List <HtmlNode>();
foreach (var searchResultComponentDiv in searchResultComponentDivs)
{
var searchResultDivList = searchResultComponentDiv.Descendants("div")
.Where(node => node.GetAttributeValue("class", "") == "rc")
.ToList();
if (searchResultDivList.Count == 0)
{
searchResultDivs.Add(searchResultComponentDiv); //anyway, vedio, img block count for 1 result
}
else
{
searchResultDivs.AddRange(searchResultDivList);
}
}
List <GoogleSearchResult> googleSearchResults = new List <GoogleSearchResult>();
for (int i = 0; i < searchResultDivs.Count; i++)
{
GoogleSearchResult googleSearchResult = new GoogleSearchResult
{
Links = new List <string>()
};
//index
googleSearchResult.Index = i + 1; //start from 1
//titles
var titleDivNode = searchResultDivs[i].Descendants("h3")
.FirstOrDefault(node => node.GetAttributeValue("class", "") == "LC20lb");
if (titleDivNode != null)
{
googleSearchResult.Title = titleDivNode.InnerText;
}
//links
var anchorNodes = searchResultDivs[i].Descendants("a")
.Where(node => node.GetAttributeValue("href", "").StartsWith("http"))
.ToList();
foreach (var anchornode in anchorNodes)
{
string link = anchornode.GetAttributeValue("href", "");
googleSearchResult.Links.Add(link);
}
//description
var descriptionDivNode = searchResultDivs[i].Descendants("div")
.FirstOrDefault(node => node.GetAttributeValue("class", "") == "s");
var descriptionNode = descriptionDivNode?.Descendants("span")
.FirstOrDefault(node => node.GetAttributeValue("class", "") == "st");
if (descriptionNode != null)
{
googleSearchResult.Description = descriptionNode.InnerText;
}
googleSearchResults.Add(googleSearchResult);
}
List <GoogleSearchResult> matchingGoogleSearchResults = new List <GoogleSearchResult>();
foreach (var googleSearchResult in googleSearchResults)
{
var links = googleSearchResult.Links;
bool shouldContinue = false;
foreach (var link in links)
{
if (link != null && link.ToLowerInvariant().Contains(matchingKeywords.ToLowerInvariant()))
{
matchingGoogleSearchResults.Add(googleSearchResult);
//here we break the current loop then continue
shouldContinue = true;
break;
}
}
if (shouldContinue)
{
continue;
}
if (googleSearchResult.Title != null && googleSearchResult.Title.ToLowerInvariant().Contains(matchingKeywords.ToLowerInvariant()))
{
matchingGoogleSearchResults.Add(googleSearchResult);
continue;
}
if (googleSearchResult.Description != null && googleSearchResult.Description.ToLowerInvariant().Contains(matchingKeywords.ToLowerInvariant()))
{
matchingGoogleSearchResults.Add(googleSearchResult);
continue;
}
}
//init response
ScrapeResponse scrapeResponse = new ScrapeResponse
{
MatchingResults = matchingGoogleSearchResults,
MatchingPositions = _scrapingService.GetPositionsFromMatchingResults(matchingGoogleSearchResults),
Message = "Scraped search result successfully!"
};
return(Ok(scrapeResponse));
}
catch (Exception ex)
{
//TODO: this should be 500 error
return(ErrorResponseBadRequest(logger.Here(), ex.Message, ErrorType.ApiExpection));
}
}
public MicrosoftDataSource(GoogleSearchResult i_Result)
{
m_Url = i_Result.Url;
GoogleResultIndex = i_Result.Index;
}
