protected override void OnInit(EventArgs e)
{
base.OnInit(e);
settings = new KeywordSettings();
settings = settings.Load(SPContext.Current.Site);
btn_Save.Click += new EventHandler(btn_Save_Click);
}
public WordScraper()
{
settings = new KeywordSettings();
settings = settings.Load();
serviceLocator = SharePointServiceLocator.GetCurrent();
serviceKeywords = serviceLocator.GetInstance <IKeywordRepository>();
}
void btn_Save_Click(object sender, EventArgs e)
{
settings = settings.Save(SPContext.Current.Site);
var eventArgsJavaScript = String.Format("{{Message:'{0}',controlIDs:window.frameElement.dialogArgs}}", "The Properties have been updated.");
((EnhancedLayoutsPage)Page).EndOperation(ModalDialogResult.OK, eventArgsJavaScript);
Context.Response.Flush();
Context.Response.End();
}
/// <summary>
/// Main entry point of the service.
/// </summary>
/// <returns></returns>
protected override ServiceOutcome DoWork()
{
// This is some crap added by Yaniv which needs to be called otherwise filenaming is screwed up
InitalizeAccountID();
// Threading mode
bool multiThreaded = false;
bool.TryParse(Instance.Configuration.Options["Multithreaded"], out multiThreaded);
TimeSpan requestDelay;
if (!TimeSpan.TryParse(Instance.Configuration.Options["DelayBetweenRequests"], out requestDelay))
{
requestDelay = TimeSpan.FromSeconds(0);
}
string urlFormat = Instance.Configuration.Options["UrlFormat"];
if (urlFormat == null)
{
throw new Exception("UrlFormat is required in the service's Options.");
}
List <KeywordSettings> keywords = new List <KeywordSettings>();
int profileID = -1;
if (Instance.ParentInstance.Configuration.Options.ContainsKey("ProfileID"))
{
if (!Int32.TryParse(Instance.ParentInstance.Configuration.Options["ProfileID"], out profileID))
{
// Invalid profile ID, report failure
throw new ArgumentException(String.Format("'{0}' is an invalid profile ID.", Instance.ParentInstance.Configuration.Options["ProfileID"]));
}
}
int defaultTotalResults;
int maxTotalResults;
int resultsPerPage;
int startOffset;
if (!Int32.TryParse(Instance.Configuration.Options["DefaultTotalResults"], out defaultTotalResults))
{
defaultTotalResults = 20;
}
if (!Int32.TryParse(Instance.Configuration.Options["MaxTotalResults"], out maxTotalResults))
{
maxTotalResults = 100;
}
if (!Int32.TryParse(Instance.Configuration.Options["ResultsPerPage"], out resultsPerPage))
{
resultsPerPage = 20;
}
if (!Int32.TryParse(Instance.Configuration.Options["StartOffset"], out startOffset))
{
startOffset = 0;
}
using (DataManager.Current.OpenConnection())
{
// Make a table of every keyword matched up with all gl/hl combinations that need to be run for it
SqlCommand seCmd = DataManager.CreateCommand(String.Format(@"
select kw.Keyword_GK as GK, kw.Keyword as Value, pkw.TotalResults as TotalResults, cse.GeoLocation as GL, cse.HostLanguage as HL
from
User_GUI_SerpProfileKeyword pkw
inner join User_GUI_SerpProfile p
on p.Profile_ID = pkw.Profile_ID {0}
inner join UserProcess_GUI_Keyword kw
on kw.Keyword_GK = pkw.Keyword_GK
inner join User_GUI_SerpProfileSearchEngine pse
on pse.Profile_ID = pkw.Profile_ID
inner join Constant_SearchEngine cse
on cse.ServiceName = @serviceName:NVarChar and
cse.Search_Engine_ID = pse.Search_Engine_ID
where
pkw.Account_ID = @accountID:Int
group by
kw.Keyword_GK, kw.Keyword, pkw.TotalResults, cse.GeoLocation, cse.HostLanguage
order by
GK, Value, GL, HL
" , profileID > -1 ? String.Format("and p.Profile_ID = {0}", profileID) : "and p.IsActive = 1"));
// Apply general parameters
seCmd.Parameters["@accountID"].Value = Instance.AccountID;
seCmd.Parameters["@serviceName"].Value = Instance.ParentInstance.Configuration.Name;
// Retrieve and add keywords
using (SqlDataReader reader = seCmd.ExecuteReader())
{
KeywordSettings prevKwSettings = null;
while (reader.Read())
{
// Either use the last kw settings (since the same kw can appear a few times in a row with different combinations)
// or create a new kw settings object
long kwGK = (long)reader["GK"];
KeywordSettings kwSettings;
if (prevKwSettings != null && kwGK == prevKwSettings.GK)
{
kwSettings = prevKwSettings;
}
else
{
int kwTotalsResults = reader["TotalResults"] is int?
(int)reader["TotalResults"] :
defaultTotalResults;
if (kwTotalsResults < resultsPerPage)
{
kwTotalsResults = resultsPerPage;
}
else if (kwTotalsResults > maxTotalResults)
{
kwTotalsResults = maxTotalResults;
}
kwSettings = new KeywordSettings(kwGK, reader["Value"] as string, kwTotalsResults);
keywords.Add(kwSettings);
prevKwSettings = kwSettings;
}
kwSettings.Combinations.Add(new string[] { reader["GL"] as string, reader["HL"] as string });
}
}
}
// Counts how many retrievals completed
int completedCount = 0;
int failedCount = 0;
string errorList = string.Empty;
// Go over each keyword and each parameter combination for each keyword
foreach (KeywordSettings keyword in keywords)
{
foreach (string[] combination in keyword.Combinations)
{
for (int s = 0; s <= keyword.TotalResults - resultsPerPage; s += resultsPerPage)
{
// Run retrievals on concurrent threads
ParameterizedThreadStart threadStart = delegate(object p)
{
object[] values = (object[])p;
KeywordSettings kw = (KeywordSettings)values[0];
string[] combo = (string[])values[1];
int start = (int)values[2];
SettingsCollection parameters = new SettingsCollection(String.Format("KeywordGK: {0}; GL: {1}; HL: {2}; Results: {3}-{4}", kw.GK, combo[0], combo[1], start + 1, start + resultsPerPage));
try
{
const int numTries = 3;
string filePath = null;
// Try numTries times to retrieve the file
for (int i = 0; i < numTries; i++)
{
try
{
filePath = WriteResultToFile(
BuildURL(urlFormat, kw.Value, combo[0], combo[1], resultsPerPage, start + startOffset),
DateTime.Now,
String.Format("{0} [{1}] {2}-{3} {4:00}-{5:00}", PathEscape(kw.Value), kw.GK, combo[0], combo[1], start + 1, start + resultsPerPage),
false);
break;
}
catch
{
if (i == numTries - 1)
{
throw;
}
else
{
Thread.Sleep(requestDelay);
}
}
}
SaveFilePathToDB(this.Instance.ParentInstance.Configuration.Name, filePath, parameters);
}
catch (Exception ex)
{
errorList += String.Format("Error on {0} ({1}): {2} - {3}\n\n", kw.Value, kw.GK, ex.GetType().Name, ex.Message);
failedCount++;
}
completedCount++;
};
Thread thread = new Thread(threadStart);
thread.Start(new object[] { keyword, combination, s });
if (!multiThreaded)
{
// When not multithreaded we have to wait for the request thread to finish
thread.Join();
if (requestDelay.TotalMilliseconds > 0)
{
Thread.Sleep(requestDelay);
}
}
}
}
}
// Check every 100 ms if the retrievals have completed
while (completedCount < keywords.Count)
{
Thread.Sleep(100);
}
if (failedCount > 0 && failedCount == completedCount)
{
Log.Write(String.Format("All {0} requests failed.\n\n{1}", completedCount, errorList), LogMessageType.Error);
return(ServiceOutcome.Failure);
}
else
{
if (failedCount > 0)
{
Log.Write(String.Format("{0} out of {1} requests failed.\n\n{2}", failedCount, completedCount, errorList), LogMessageType.Error);
}
return(ServiceOutcome.Success);
}
}
