public void Handle(YahooHeatData heatData, Logger logger)
{
try
{
using (var ctx = new HeatYahooContext())
{
ctx.BulkInsert(new YahooHeatData[] {heatData});
}
}
catch (DbEntityValidationException e)
{
logger.Fatal(e.Dump());
throw;
}
catch (SqlException e)
{
if (!e.Message.Contains("Cannot insert duplicate key row"))
{
logger.Fatal(e.Dump());
throw;
}
}
catch (Exception e)
{
logger.Fatal(e.Dump());
throw;
}
}
public void ParseEmlFiles(CancellationToken token)
{
var emails = new DirectoryInfo(Settings.WorkingDirectory).GetFiles();
if (emails.Any())
{
if (Settings.ParserMaxBatchSize > 0 && emails.Count() > Settings.ParserMaxBatchSize)
{
emails = emails.Take(Settings.ParserMaxBatchSize).ToArray();
}
foreach (var eml in emails)
{
if (token.IsCancellationRequested)
{
break;
}
var yahooHeatData = new YahooHeatData() {IsFirst = true, ServerId = Settings.ServerID};
var archiveFolder = Path.Combine(Settings.ArchiveDirectory, DateTime.Now.ToString("yyyyMMdd"));
if (!Directory.Exists(archiveFolder))
{
log.Info("Creating archive folder: " + archiveFolder);
Directory.CreateDirectory(archiveFolder);
}
try
{
log.Info("Parsing email " + eml.FullName);
var emailMessage = new MailBuilder().CreateFromEmlFile(eml.FullName);
yahooHeatData.RawText = emailMessage.RenderEml(true);
yahooHeatData.HeatTime = DateTime.Parse(emailMessage.Document.Root.Headers["x-heat-time"]);
yahooHeatData.heat_ip = emailMessage.Document.Root.Headers["x-heat-ip"];
const string fromRegex = @"\""?(?<fromname>.+)""?\s+<(?<email>.+)>";
var match = Regex.Match(emailMessage.Document.Root.Headers["from"], fromRegex);
var fromAddress = new MailAddress(match.Groups["email"].ToString(),
match.Groups["fromname"].ToString());
// FROM
//TODO: Where do these go?
//yahooHeatData.from = emailMessage.Document.Root.Headers["from"].Replace("\"", string.Empty);
yahooHeatData.fromnamedisplay = fromAddress.DisplayName.Replace("\"", "");
//yahooHeatData.FromAddress = fromAddress.Address;
yahooHeatData.fromemailaddress_user = fromAddress.User;
yahooHeatData.fromemailaddress_host = fromAddress.Host;
// TO
yahooHeatData.toemailaddress = emailMessage.Document.Root.Headers["to"];
var recipient = new MailAddress(emailMessage.Document.Root.Headers["to"]);
string recipientDomain = recipient.Host;
// BULK/INBOX
if (recipientDomain == "yahoo.com" || recipientDomain == "ymail.com" ||
recipientDomain == "rocketmail.com" ||
recipientDomain == "yahoo.co.uk" || recipientDomain == "att.net")
{
yahooHeatData.foldername =
string.IsNullOrEmpty(emailMessage.Document.Root.Headers["x-yahoofilteredbulk"])
? "INBOX"
: "BULK";
}
else if (recipientDomain == "aol.com")
{
yahooHeatData.foldername = emailMessage.Document.Root.Headers["x-heat-folder"].ToLower() ==
"inbox"
? "INBOX"
: "BULK";
}
yahooHeatData.subject = emailMessage.Document.Root.Headers["subject"];
// SENT TIME
yahooHeatData.DateSent = DateTime.Parse(emailMessage.Document.Root.Headers["date"]);
var receivedRegex = string.Empty;
if (recipientDomain == "yahoo.com" || recipientDomain == "ymail.com" ||
recipientDomain == "rocketmail.com" ||
recipientDomain == "yahoo.co.uk" || recipientDomain == "att.net")
{
receivedRegex =
@"\(EHLO\s(?<ehlodomain>[a-zA-Z0-9\-\.]+)\)\s+\((?<ehloip>(?:[0-9]{1,3}.?){4})\)\s+by\s+(?<receivedmx>[a-zA-Z0-9\.\-]+)\s+with\s+SMTP;\s+(?<receiveddate>.+)";
}
else if (recipientDomain == "aol.com")
{
receivedRegex =
@"\((?<ehlodomain>[a-zA-Z0-9\-\.]+)\s+\[(?<ehloip>(?:[0-9]{1,3}\.?){4})\]\)\s+by\s+(?<receivedmx>[a-zA-Z0-9\.\-]+)\s.+;\s(?<receiveddate>.+)\s\(";
}
bool matched = false;
foreach (var headerValue in emailMessage.Document.Root.Headers.GetValues("received"))
{
match = Regex.Match(headerValue, receivedRegex,
RegexOptions.ExplicitCapture | RegexOptions.IgnoreCase);
if (match.Success)
{
yahooHeatData.ehlodomain = match.Groups["ehlodomain"].ToString(); // ehlo domain
yahooHeatData.ehloip = match.Groups["ehloip"].ToString(); // ehlo ip
yahooHeatData.receivedmx = match.Groups["receivedmx"].ToString(); // received mx
yahooHeatData.DateReceived = DateTime.Parse(match.Groups["receiveddate"].ToString());
matched = true;
break;
}
}
if (!matched)
{
throw new Exception("Failed to find EHLO info from \"received\" headers");
}
// SPF
string spfHeader = string.Empty;
string spfRegex = string.Empty;
if (recipientDomain == "yahoo.com" || recipientDomain == "ymail.com" ||
recipientDomain == "rocketmail.com"
|| recipientDomain == "yahoo.co.uk" || recipientDomain == "att.net")
{
spfRegex = @"^(\w+)";
spfHeader = "received-spf";
}
else if (recipientDomain == "aol.com")
{
spfRegex = @"SPF : (\w+)";
spfHeader = "x-aol-spf";
}
if (!string.IsNullOrWhiteSpace(emailMessage.Document.Root.Headers[spfHeader]))
{
match = Regex.Match(emailMessage.Document.Root.Headers[spfHeader], spfRegex);
yahooHeatData.spfheader = match.Groups[1].ToString();
}
// DomainKey
string domainkeysHeader;
string domainkeysRegex;
if (recipientDomain == "yahoo.com" || recipientDomain == "ymail.com" ||
recipientDomain == "rocketmail.com"
|| recipientDomain == "yahoo.co.uk" || recipientDomain == "att.net")
{
domainkeysHeader = "Authentication-Results";
domainkeysRegex = @"domainkeys=(\w+)";
match = Regex.Match(emailMessage.Document.Root.Headers[domainkeysHeader], domainkeysRegex);
yahooHeatData.domainkeysAuthenticationResults = match.Groups[1].ToString();
}
// DKIM
string dkimHeader = string.Empty;
string dkimRegex = string.Empty;
if (recipientDomain == "yahoo.com" || recipientDomain == "ymail.com" ||
recipientDomain == "rocketmail.com"
|| recipientDomain == "yahoo.co.uk" || recipientDomain == "att.net")
{
dkimHeader = "Authentication-Results";
dkimRegex = @"dkim=(\w+)";
}
else if (recipientDomain == "aol.com")
{
dkimHeader = "x-aol-scoll-authentication";
dkimRegex = @"DKIM\s:\s(\w+)";
}
if (!string.IsNullOrWhiteSpace(emailMessage.Document.Root.Headers[dkimHeader]))
{
match = Regex.Match(emailMessage.Document.Root.Headers[dkimHeader], dkimRegex);
yahooHeatData.dkimAuthenticationResults = match.Groups[1].ToString();
}
// Recipient domain
yahooHeatData.recipientDomain = recipientDomain;
const string unsubscribeHeader = "list-unsubscribe";
// Base 26 Info
if (!string.IsNullOrWhiteSpace(emailMessage.Document.Root.Headers[unsubscribeHeader]))
{
// Batch-Subscriber-List Id
IDictionary<string, int> base26Values;
MailingSystem mailingSystem = DetermineMailingSystemFromListUnsubscribe(emailMessage.Document.Root.Headers[unsubscribeHeader]);
switch (mailingSystem)
{
case MailingSystem.WhiteDelivery:
base26Values = GetWdBase26Info(emailMessage.Document.Root.Headers[unsubscribeHeader]);
yahooHeatData.BatchId = base26Values["batch_id"];
yahooHeatData.SubscriberID = base26Values["subscriber_id"];
yahooHeatData.ListID = base26Values["list_id"];
break;
case MailingSystem.Avenlo:
base26Values = GetAvenloBase26Info(emailMessage.Document.Root.Headers[unsubscribeHeader]);
yahooHeatData.AvenloDeploymentId = base26Values["action_id"];
yahooHeatData.AvenloRecipientId = base26Values["redirect_id"];
break;
}
}
const string apperentlyToHeader = "x-apparently-to";
if (!string.IsNullOrWhiteSpace(emailMessage.Document.Root.Headers[apperentlyToHeader]))
{
const string apparentlyToRegex =
@"(?<apparentFromEmail>.+)\s+via\s+(?<apparentReceivedIp>.+);\s+(?<apparentReceivedDate>.+)";
match = Regex.Match(emailMessage.Document.Root.Headers[apperentlyToHeader],
apparentlyToRegex);
yahooHeatData.received_ip = match.Groups["apparentReceivedIp"].ToString();
}
if (!PublishMessage(yahooHeatData))
{
if (Settings.ParserBatchWaitTime < MinimumWaitSecondsOnQueueConnectionFailure)
{
System.Threading.Thread.Sleep(MinimumWaitSecondsOnQueueConnectionFailure -
Settings.ParserProcessingDelay);
}
return;
}
// archive file
string archiveFile = Path.Combine(archiveFolder, eml.Name);
eml.MoveTo(archiveFile);
log.Debug("Archived file: " + archiveFile);
}
catch (Exception ex)
{
log.Error("Failed to parse: " + eml.FullName);
log.Error(ex.ToString());
eml.MoveTo(Path.Combine(Settings.ErrorDirectory, eml.Name));
continue;
}
if (Settings.ParserProcessingDelay > 0)
{
System.Threading.Thread.Sleep(Settings.ParserProcessingDelay);
}
}
}
else
{
log.Info("No emails to parse");
}
}
public bool PublishMessage(YahooHeatData data)
{
try
{
if (_bus == null)
{
_bus = RabbitHutch.CreateBus(ConfigurationManager.AppSettings["Bus"]);
}
_bus.Publish(data, "YahooHeatData");
rabbitDownSince = null;
return true;
}
catch (Exception ex)
{
if (!rabbitDownSince.HasValue)
{
rabbitDownSince = DateTime.Now;
log.Fatal("Failed to connect to local queue", ex);
}
else if (rabbitDownSince.Value.AddMinutes(WaitTimeForQueueRecovery) > DateTime.Now)
{
log.Fatal("RabbitMQ reconnect attempts failed " + Settings.ServerID, ex);
throw new QueueConnectionException(
string.Format("Rabbit connection {0}", WaitTimeForQueueRecovery), ex);
}
else if (DateTime.Now.Subtract(rabbitDownSince.Value).TotalMinutes > 5 &&
DateTime.Now.Subtract(rabbitLastAlerted).TotalMinutes > 5)
{
log.Fatal("Failed to connect to local queue " + Settings.ServerID, ex);
}
else
{
log.Error("Failed to connect to local queue " + Settings.ServerID, ex);
}
return false;
}
}
private void ProcessYahoo(YahooHeatData heatData)
{
new YahooDataHandler().Handle(heatData, _logger);
}
