public void SetResponse(SerializableWebResponse response)
{
Regex rx;
MatchCollection matches;
string captureString;
rx = new Regex(REGEX_DIGGS);
matches = rx.Matches(response.Content);
if (matches.Count == 0)
{
_errorState = true;
return;
}
captureString = matches[0].Groups[1].Value;
_diggCount = int.Parse(captureString);
rx = new Regex(REGEX_COMMENT_COUNT);
matches = rx.Matches(response.Content);
if (matches.Count == 0)
{
_errorState = true;
return;
}
_commentCount = int.Parse(matches[0].Groups[1].Value);
_finished = true;
}
public void SetResponse(SerializableWebResponse response)
{
_log.DebugFormat("Parsing PageRank response: '{0}'", response.ToString());
_pageRank = _gpr.ParseRankFromResult(response.Content);
_log.DebugFormat("PageRank parsed as {0}", _pageRank);
}
public void SetResponse(SerializableWebResponse response)
{
switch (_currentStage)
{
case stage.GetHiddenFields:
//Advance to the next stage
_currentStage = stage.Authenticate;
processLoginFormHiddenFields(response);
break;
case stage.Authenticate:
//Advance to the next stage
_currentStage = stage.PostAuthenticateRedirect;
processSendCredentialsResponse(response);
break;
case stage.PostAuthenticateRedirect:
//Advance to the next stage
_currentStage = stage.RunReports;
processPostAuthenticateResponse(response);
break;
case stage.RunReports:
processReportResponse(response);
break;
}
}
/// <summary>
/// Parses the response and stores the results in the database.
/// </summary>
/// <remarks>
/// This should be called once the request has been made.
/// </remarks>
/// <param name="response">
/// The response that was recieved as a result of making the
/// request from <see cref="SerializableWebRequest"/>.
/// </param>
public void SetResponse(SerializableWebResponse response)
{
XmlSerializer serializer;
StringReader sr;
YTech.General.Web.Seo.RankChecking.Yahoo.ResultSet rs;
using (sr = new StringReader(response.Content))
{
serializer = new XmlSerializer(typeof(ResultSet));
rs = (ResultSet)serializer.Deserialize(sr);
}
for (int i = 0; i < rs.Result.Length; i++)
{
if (rs.Result[i].Url.StartsWith(Url, true, CultureInfo.InvariantCulture))
{
//We found a match!
//Remember, item 0 in the array is search result #1
_searchPosition = i + 1;
break;
}
}
}
public void SetResponse()
{
SerializableWebResponse resp;
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("Delicious_Sample1.txt");
d.SetResponse(resp);
}
public bool ReprocessItem(int queueId)
{
DatasourceManager datasourceManager;
DatasourceDataQueueItem queueItem;
RawDataValue rawData;
ConfiguredDatasource cds;
IDataSource currentDatasource;
DatasourceParameterType[] parameterTypes;
queueItem = _db.ORManager.Get <DatasourceDataQueueItem>(queueId);
rawData = _db.ORManager.Get <RawDataValue>(queueItem.RawDataId);
cds = _db.ORManager.Get <ConfiguredDatasource>(rawData.ConfiguredDatasourceId);
datasourceManager = new DatasourceManager();
currentDatasource = datasourceManager.GetDatasource(cds.DatasourceType.Id);
//Load the parameter types
parameterTypes = _db.GetParameterTypesForDatasource(cds.DatasourceType.Id);
//Set up the datasource
cds.InitializeDatasource(currentDatasource, parameterTypes);
//Fake the web response
SerializableWebResponse resp;
resp = new SerializableWebResponse();
resp.Content = queueItem.Data;
currentDatasource.SetResponse(resp);
RawDataValue[] values;
values = currentDatasource.Values;
foreach (RawDataValue currValue in values)
{
if (!currValue.Success || currValue.Fuzzy)
{
return(false);
}
}
//Save the new values
foreach (RawDataValue currValue in values)
{
currValue.ConfiguredDatasourceId = cds.Id.Value;
_db.BulkInsertRawData(values);
}
//Delete the extra state data
_db.ORManager.Delete(queueItem);
//If we get here, it must have worked!
return(true);
}
private void processLoginFormHiddenFields(SerializableWebResponse resp)
{
MatchCollection matches;
matches = Regex.Matches(resp.Content, "<input type=\"hidden\" name=\"(.*?)\" value=\"(.*?)\">");
_loginFormHiddenFields = new string[matches.Count][];
for (int i = 0; i < matches.Count; i++)
{
_loginFormHiddenFields[i] = new string[2];
_loginFormHiddenFields[i][0] = matches[i].Groups[1].Value;
_loginFormHiddenFields[i][1] = matches[i].Groups[2].Value;
}
}
public void SetResponse_Fuzzy()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults_Fuzzy.txt");
gr.Url = "blah";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(true, values[0].Fuzzy);
}
public void SetResponse(SerializableWebResponse response)
{
Regex re;
MatchCollection matches;
int resultCount;
re = new Regex(SEARCH_RESULT_REGEX);
matches = re.Matches(response.Content);
//Check if the page has no results
if (response.Content.Contains("did not match any documents"))
{
_searchPosition = null;
return;
}
//Check the number of results to make sure our regex isn't broken
resultCount = getSearchResultCount(response.Content);
if (matches.Count != resultCount)
{
_log.WarnFormat("There were {0} matches, but the page says there should be {1} results", matches.Count, resultCount);
_log.DebugFormat("Dumping response for analysis: {0}", response.Content);
if (matches.Count >= resultCount - 3 && matches.Count <= resultCount + 3)
{
_fuzzy = true;
}
else
{
_log.Debug("The result is too far off to be considered fuzzy");
_errorState = true;
return;
}
}
for (int i = 0; i < matches.Count; i++)
{
if (matches[i].Groups[1].Value.StartsWith(Url, true, CultureInfo.InvariantCulture))
{
//we found a match
//Remember, item 0 in the array is search result #1
_searchPosition = i + 1;
break;
}
}
}
public void ErrorHandling()
{
RawDataValue[] values;
d.Parameters.Add(1, "http://www.digg.com/asdfasdf");
req = d.GetNextRequest();
resp = new SerializableWebResponse();
resp.Content = "asdfasdf";
d.SetResponse(resp);
values = d.Values;
Assert.AreEqual(2, values.Length);
Assert.AreEqual(false, values[0].Success);
Assert.AreEqual(false, values[1].Success);
}
public void SetResponse3()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults3.txt");
gr.Url = "http://www.kbalertz.com/kb_933568.aspx";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(2, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse7()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults7.txt");
gr.Url = "http://find.yuku.com/find/communities?q=survivorsucks+teens";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(11, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse5()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults5.txt");
gr.Url = "http://en.wikipedia.org/wiki/Scuba_diving";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(5, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse6()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults6.txt");
gr.Url = "maps.caribseek.com/";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(4, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse4()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults4.txt");
gr.Url = "http://www.obishawn.com/archive/2006/02/19/4.aspx";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(7, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse2()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults2.txt");
gr.Url = "http://www.humanmetrics.com/cgi-win/JTypes1.htm";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(4, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse_BadResults()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults_Bad.txt");
gr.Url = "http://www.blackwell-synergy.com";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(null, values[0].FloatValue);
Assert.AreEqual(false, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse_8Results()
{
SerializableWebResponse sr;
sr = new SerializableWebResponse();
sr.Content = readEmbeddedText("GoogleRankResults_8Results.txt");
gr.Url = "http://www.ObiShawn.com";
gr.SetResponse(sr);
values = gr.Values;
Assert.AreEqual(1, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(false, values[0].Fuzzy);
}
public void SetResponse()
{
SerializableWebResponse resp;
RawDataValue[] values;
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("GoogleBacklinks_Sample1.txt");
gb.SetResponse(resp);
values = gb.Values;
Assert.AreEqual(1, values.Length);
Assert.AreEqual(null, values[0].DatasourceSubTypeId);
Assert.AreEqual(86, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.IsTrue(values[0].Timestamp > DateTime.UtcNow.AddSeconds(-10) && values[0].Timestamp <= DateTime.UtcNow);
}
public void SetResponse2()
{
SerializableWebResponse resp;
RawDataValue[] values;
resp = new SerializableWebResponse();
resp.Content = "asdfasd";
gb.SetResponse(resp);
values = gb.Values;
Assert.AreEqual(1, values.Length);
Assert.AreEqual(null, values[0].DatasourceSubTypeId);
Assert.AreEqual(null, values[0].FloatValue);
Assert.AreEqual(false, values[0].Success);
Assert.IsTrue(values[0].Timestamp > DateTime.UtcNow.AddSeconds(-10) && values[0].Timestamp <= DateTime.UtcNow);
}
public void Comments()
{
RawDataValue[] values;
d.Parameters.Add(1, "http://www.digg.com/asdfasdf");
req = d.GetNextRequest();
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("Digg_Sample2.txt");
d.SetResponse(resp);
values = d.Values;
Assert.AreEqual(2, values.Length);
Assert.AreEqual(6, values[1].DatasourceSubTypeId);
Assert.AreEqual(16, values[1].FloatValue);
Assert.AreEqual(true, values[1].Success);
Assert.IsTrue(values[1].Timestamp > DateTime.UtcNow.AddSeconds(-10) && values[1].Timestamp <= DateTime.UtcNow);
}
public SerializableWebResponse WrapHttpResponse(HttpWebResponse httpResponse)
{
StreamReader sr;
SerializableWebResponse response;
_log.Debug("Wrapping HttpWebResponse stream into its serializable form");
using (sr = new StreamReader(httpResponse.GetResponseStream()))
{
response = new SerializableWebResponse();
response.Content = sr.ReadToEnd();
}
#if (DEBUG)
_log.DebugFormat("Response contained {0}", response.Content);
#endif
return(response);
}
private void processSendCredentialsResponse(SerializableWebResponse resp)
{
Regex reg;
MatchCollection urlMatches;
RegexOptions options;
options = RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.CultureInvariant;
reg = new Regex("<a href=\"(https://.*?)\"", options);
urlMatches = reg.Matches(resp.Content);
//Check if we found the URL to post to
if (urlMatches.Count == 0)
{
throw new ApplicationException("Unable to find redirect URL in Google Adsense authetication response");
}
_authenticateRedirectUrl = urlMatches[0].Groups[1].Value;
}
public void SetResponse()
{
SerializableWebResponse resp;
RawDataValue[] values;
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("MSLiveSearch_Sample1.txt");
m.Parameters = new Dictionary <int, object>();
m.Parameters.Add(1, "test search");
m.Url = "http://www.speakeasy.net/speedtest/";
m.GetNextRequest();
m.SetResponse(resp);
values = m.Values;
Assert.AreEqual(1, values.Length);
Assert.AreEqual(3, values[0].FloatValue);
Assert.AreEqual(true, values[0].Success);
Assert.IsTrue(values[0].Timestamp > DateTime.UtcNow.AddSeconds(-10) && values[0].Timestamp <= DateTime.UtcNow);
}
public void SetResponse(SerializableWebResponse response)
{
_position = GetSearchPosition(response.Content, _url);
}
private void processReportResponse(SerializableWebResponse response)
{
_values = ParseAdsenseCsvReport(response.Content, _channelName);
}
private void processPostAuthenticateResponse(SerializableWebResponse resp)
{
//The login process is now complete
}
public void SetResponse(SerializableWebResponse response)
{
_linkCount = GetLinkCount(response.Content);
}
public void Sample1()
{
GoogleAdsense ga;
SerializableWebRequest req;
SerializableWebResponse resp;
ga = new GoogleAdsense();
ga.Parameters = new Dictionary <int, object>();
ga.Parameters.Add(1, "testUser");
ga.Parameters.Add(2, "testPass");
//Stage 1
req = ga.GetNextRequest();
Assert.AreEqual(null, req.PostData);
Assert.AreEqual("https://www.google.com/accounts/ServiceLoginAuth?service=adsense&hl=en-US<mpl=login&ifr=true&passive=true&rm=hide&nui=3&alwf=true&continue=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth&followup=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth", req.Url);
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("GoogleAdsense_Sample1_Stage1.txt");
ga.SetResponse(resp);
//Stage 2
req = ga.GetNextRequest();
Assert.AreEqual("https://www.google.com/accounts/ServiceLoginAuth", req.Url);
Assert.AreEqual("ltmpl=login&continue=https%3a%2f%2fwww.google.com%2fadsense%2fgaiaauth&followup=https%3a%2f%2fwww.google.com%2fadsense%2fgaiaauth&service=adsense&nui=3&ifr=true&rm=hide<mpl=login&hl=en-US&alwf=true&Email=testUser&Passwd=testPass&null=Sign+in", req.PostData);
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("GoogleAdsense_Sample1_Stage2.txt");
ga.SetResponse(resp);
//Stage 3
req = ga.GetNextRequest();
Assert.AreEqual("https://www.google.com/accounts/CheckCookie?continue=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth&followup=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth&service=adsense&hl=en-US&chtml=LoginDoneHtml", req.Url);
Assert.AreEqual(null, req.PostData);
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("GoogleAdsense_Sample1_Stage3.txt");
ga.SetResponse(resp);
//Stage 4
req = ga.GetNextRequest();
Assert.AreEqual("https://www.google.com/adsense/report/aggregate?dateRange.dateRangeType=custom&dateRange.customDate.start.month=" + DateTime.Now.AddDays(-1).Date.Month + "&dateRange.customDate.start.day=" + DateTime.Now.AddDays(-1).Date.Day + "&dateRange.customDate.start.year=" + DateTime.Now.AddDays(-1).Date.Year + "&dateRange.customDate.end.month=" + DateTime.Now.AddDays(-1).Month + "&dateRange.customDate.end.day=" + DateTime.Now.AddDays(-1).Date.Day + "&dateRange.customDate.end.year=" + DateTime.Now.AddDays(-1).Date.Year + "&outputFormat=TSV_EXCEL", req.Url);
Assert.AreEqual(null, req.PostData);
resp = new SerializableWebResponse();
resp.Content = readEmbeddedText("GoogleAdsense_Sample1_Stage4.txt");
ga.SetResponse(resp);
//Check data
RawDataValue[] values;
values = ga.Values;
Assert.AreEqual(3, values.Length);
Assert.AreEqual(1, values[0].DatasourceSubTypeId);
Assert.AreEqual(true, values[0].Success);
Assert.AreEqual(1.64, values[0].FloatValue);
Assert.AreEqual(DateTime.Parse("2-22-07"), values[0].Timestamp);
Assert.AreEqual(2, values[1].DatasourceSubTypeId);
Assert.AreEqual(true, values[1].Success);
Assert.AreEqual(616, values[1].FloatValue);
Assert.AreEqual(DateTime.Parse("2-22-07"), values[1].Timestamp);
Assert.AreEqual(4, values[2].DatasourceSubTypeId);
Assert.AreEqual(true, values[2].Success);
Assert.AreEqual(16, values[2].FloatValue);
Assert.AreEqual(DateTime.Parse("2-22-07"), values[2].Timestamp);
}
