private static IDownloader GenerateDownloader(RetailerConfiguration retailer)
{
try
{
switch (retailer.RetailerName)
{
case "Cel":
{
return(new CelDownloader(retailer));
}
case "PcGarage":
{
return(new PcGarageDownloader(retailer));
}
default:
{
return(null);
}
}
}
catch (Exception ex)
{
GenericLogger.Error($"could not create downloader for {retailer.RetailerName}");
return(null);
}
}
private void ParseProducts(DbModelContext dbContext)
{
int counter = 0;
do
{
Thread.Sleep(3 * 1000);
if (products.Count > 0)
{
var currentProduct = products[counter];
parser.GetProduct(ref currentProduct);
counter++;
try
{
dbContext.SaveChanges();
}
catch (Exception ex)
{
GenericLogger.Error($"Error when trying to save changes for updating process for {currentProduct.Url}, \n ex : {ex.Message} \n {ex.StackTrace} \n {ex.InnerException.InnerException.Message}");
}
}
} while (counter < products.Count);
}
/// <summary>
/// Calls the CredUIConfirmCredentials function which confirms that the credentials
/// provided for the target should be kept in the case they are persisted.</summary>
/// <param name="value">A value of true if the credentials should be persisted.</param>
public void Confirm(bool value)
{
Credmgmtui.ReturnCodes res = Credmgmtui.ConfirmCredentials(this.Target, value);
if (res != Credmgmtui.ReturnCodes.NoError &&
res != Credmgmtui.ReturnCodes.ErrorInvalidParameter)
{
GenericLogger <CredPrompt> .Error(
LocalizibleStrings.ConfirmationFailed + res);
}
}
internal void DownloadProduct(ref Produs product)
{
try
{
CreateProductDocument(product);
ExtractNeededInformation(ref product);
}
catch (Exception ex)
{
GenericLogger.Error($"Exception popped when trying to extract product information for {Url}", ex);
}
}
public static void CrawlStart(CrawlSettingsModel model)
{
try
{
Parallel.ForEach
(model.Retailers,
new ParallelOptions()
{
MaxDegreeOfParallelism = 2
},
m => StartCrawling(m)
);
}
catch (Exception ex)
{
GenericLogger.Error("Exception thrown at MainCrawlStarter", ex);
}
}
public void DownloadBlob(string fileName, string container, string usage)
{
string path = Path.Combine(Environment.CurrentDirectory, "BlobFiles", usage);
try
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer containerRef = blobClient.GetContainerReference(container);
CloudBlockBlob blockBlob = containerRef.GetBlockBlobReference(fileName);
Directory.CreateDirectory(path);
blockBlob.DownloadToFile(Path.Combine(path, fileName), FileMode.Create);
}
catch (Exception ex)
{
GenericLogger.Error($"Exception {ex.Message} popped when trying to get Blob : {fileName} from container : {container}", ex);
}
}
public static string GetHttpRequestResponse(string url)
{
Thread.Sleep(2000);
string html = string.Empty;
try
{
html = client.GetStringAsync(url).Result;
}
catch (HttpRequestException requestException)
{
GenericLogger.Error($"Http Exception Popped for {url} when trying to GetHttpRequestResponse(string url)", requestException);
}
catch (Exception ex)
{
GenericLogger.Error("Generic exception Popped in GetHttpRequestResponse(string url)", ex);
}
return(html);
}
public static CrawlSettingsModel LoadCrawlSettings(string usage)
{
AzureBlobUtils.Instance.DownloadBlob("CrawlingTags.json", "crawlsettings", usage);
try
{
string text = string.Empty;
using (StreamReader reader = new StreamReader(Path.Combine(Environment.CurrentDirectory, "BlobFiles", usage, "CrawlingTags.json")))
{
text = reader.ReadToEnd();
}
model = JsonUtilities.Instance.DeserializeJson <CrawlSettingsModel>(text);
GenericLogger.Info($"settings for {usage} successfully downloaded");
}
catch (Exception ex)
{
GenericLogger.Error($"Exception {ex.Message} popped when trying to parse crawling Settings", ex);
}
return(model);
}
public static string GetWebRequestResponse(string url)
{
string html = string.Empty;
try
{
Uri uri = new Uri(url);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
request.Accept = "text/html, */*";
request.MaximumAutomaticRedirections = 10;
request.AllowAutoRedirect = true;
request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36";
request.Host = uri.Host;
request.Accept = "Accept: text / html,application / xhtml + xml,application / xml; q = 0.9,image / webp,image / apng,*/*;q=0.8";
request.Headers["Accept-Encoding"] = "gzip, deflate";
request.Headers["Accept-Language"] = "en-US,en;q=0.9,ro;q=0.8";
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
html = reader.ReadToEnd();
}
}
}
}
catch (WebException webEx)
{
GenericLogger.Error($"WebException popped when tring to extract {url}, \n Ex : {webEx.Message} \n {webEx.StackTrace}");
}
catch (Exception ex)
{
GenericLogger.Error($"Exception {ex.Message} popped when trying to get source for {url}, \n \n {ex.StackTrace}");
}
return(html);
}
public void SaveProducts(string givenMessage)
{
using (DbModelContext dbContext = new DbModelContext())
{
string searchCode = GetSearchCode(givenMessage);
foreach (var item in this)
{
try
{
if (dbContext.Produs.Any(prd => prd.Id.Equals(item.Id)))
{
dbContext.EvolutiaPretului.Add(item.EvolutiaPretului.OrderByDescending(m => m.Data_Actualizare).First());
}
else
{
dbContext.Produs.Add(item);
}
dbContext.AparitieProdus.Add(new AparitieProdus
{
Id_Cautare = StringToGuid(givenMessage)
,
Id_Produs = item.Id,
Id = StringToGuid($"{StringToGuid(givenMessage).ToString()}{item.Id.ToString()})")
});
dbContext.SaveChanges();
}
catch (Exception ex)
{
GenericLogger.Error($"Batch failed to insert {ex.Message} \n {ex.InnerException.InnerException.Message}");
}
}
}
}
public void GenEmails()
{
try
{
if (this.senders.Count <= 0)
{
GenericLogger <Generator> .Error(LocalizibleStrings.SendersCount);
return;
}
if (this.recipients.Count <= 0)
{
GenericLogger <Generator> .Error(LocalizibleStrings.RecipientsCount);
return;
}
ParallelQuery <GenResult> results = this.recipients.AsParallel().SelectMany(recipient =>
{
List <GenResult> genMsgs =
new List <GenResult>(this.countPerRecipient);
int genCount = 0;
while (genCount < this.countPerRecipient)
{
foreach (StoredContact sender in this.senders)
{
foreach (Storage.Message template in this.templates)
{
GenResult newMsg =
this.EmailMessageFromMessage(
sender,
recipient,
template);
if (newMsg.msg == null)
{
return(genMsgs);
}
genMsgs.Add(newMsg);
genCount++;
if (newMsg.reply != null)
{
genCount++;
}
if (newMsg.response != null)
{
genCount++;
}
GenericLogger <Generator> .Message(
LocalizibleStrings.MsgCreatedOf,
genCount, this.countPerRecipient);
this.callback.BeginInvoke(null, null);
if (genCount == this.countPerRecipient)
{
return(genMsgs);
}
}
if (genCount == this.countPerRecipient)
{
return(genMsgs);
}
}
}
return(genMsgs);
});
GenericLogger <Generator> .Message(LocalizibleStrings.MsgSendingStarted);
results.ForAll(res =>
{
try
{
res.msg.Send();
if (res.response != null)
{
res.response.Send();
}
if (res.reply != null)
{
res.reply.Send();
}
}
catch (Exception exc)
{
GenericLogger <Generator> .Message("Sending error " + exc);
}
});
GenericLogger <Generator> .Message(LocalizibleStrings.MsgSendingFinished);
}
catch (Exception exc)
{
GenericLogger <Generator> .Error(
LocalizibleStrings.ErrorGenMailMsg, exc);
}
}
private GenResult EmailMessageFromMessage(
StoredContact sender,
StoredContact recipient,
Storage.Message msg)
{
GenResult res = new GenResult();
try
{
EmailAddress em =
Contact.Bind(this.service, new ItemId(sender.UniqId))
.EmailAddresses[EmailAddressKey.EmailAddress1];
EmailMessage newMsg = new EmailMessage(this.service)
{
From = em,
Sender = em,
Body = new MessageBody(msg.BodyHtml),
Subject = msg.Subject
};
newMsg.Save(WellKnownFolderName.Drafts);
foreach (object obj in msg.Attachments)
{
Storage.Attachment attach = obj as Storage.Attachment;
if (attach == null)
{
continue;
}
newMsg.Attachments.AddFileAttachment(attach.FileName, attach.Data);
}
this.FillAdresses(ref newMsg, recipient.Email);
res.msg = newMsg;
// делаем ли форвард
if (RndTrueFalse())
{
newMsg.Update(ConflictResolutionMode.AlwaysOverwrite);
ResponseMessage respMsg = newMsg.CreateForward();
respMsg.BodyPrefix = @"test body prefix for forward message";
this.FillAdressesForResponse(ref respMsg, this.RndRecipMail());
respMsg.Save(WellKnownFolderName.Drafts);
res.response = respMsg;
}
// делаем ли реплай
if (RndTrueFalse())
{
/*newMsg.ReplyTo.Add(_RndRecipMail());
* if (_RndTrueFalse())
* {
* newMsg.ReplyTo.Add(_RndRecipMail());
* }
*
* if (_RndTrueFalse()) */
{
newMsg.Update(ConflictResolutionMode.AlwaysOverwrite);
ResponseMessage replMsg = newMsg.CreateReply(RndTrueFalse());
replMsg.BodyPrefix = @"test body prefix for reply message";
this.FillAdressesForResponse(ref replMsg, recipient.Email);
replMsg.Save(WellKnownFolderName.Drafts);
res.reply = replMsg;
}
}
}
catch (Exception exc)
{
GenericLogger <Generator> .Error(
LocalizibleStrings.ErrorCreateFromTemplate + msg.FileName, exc);
}
return(res);
}
