private static void StartSearch(CloudQueueMessage message, CrawlSettingsModel model)
{
GenericLogger.Info($"Started searching for message {message.AsString}");
Parallel.ForEach(model.Retailers
, new ParallelOptions()
{
MaxDegreeOfParallelism = 2
}
, retailer => SearchMessageCrawler.StartMessageCrawling(message, retailer)
);
}
public static void StartSearching(CrawlSettingsModel model)
{
if (AzureQueueUtils.Instance.QueueHasItems())
{
var message = AzureQueueUtils.Instance.GetQueueItem();
StartSearch(message, model);
}
else
{
//GenericLogger.Info("Queue Has NO items to be searched");
}
}
public static void CrawlStart(CrawlSettingsModel model)
{
try
{
Parallel.ForEach
(model.Retailers,
new ParallelOptions()
{
MaxDegreeOfParallelism = 2
},
m => StartCrawling(m)
);
}
catch (Exception ex)
{
GenericLogger.Error("Exception thrown at MainCrawlStarter", ex);
}
}
public static CrawlSettingsModel LoadCrawlSettings(string usage)
{
AzureBlobUtils.Instance.DownloadBlob("CrawlingTags.json", "crawlsettings", usage);
try
{
string text = string.Empty;
using (StreamReader reader = new StreamReader(Path.Combine(Environment.CurrentDirectory, "BlobFiles", usage, "CrawlingTags.json")))
{
text = reader.ReadToEnd();
}
model = JsonUtilities.Instance.DeserializeJson <CrawlSettingsModel>(text);
GenericLogger.Info($"settings for {usage} successfully downloaded");
}
catch (Exception ex)
{
GenericLogger.Error($"Exception {ex.Message} popped when trying to parse crawling Settings", ex);
}
return(model);
}
