public void ProviderShouldBeOverridenWhenSpfandDkimFailAndOnBlocklist(DmarcResult spfResult, DmarcResult dkimResult, int blocklistCount, string expectedProvider)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(spfResult, dkimResult, blockListCount: blocklistCount);
List <DomainDateProviderIpRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderIpRecord();
Assert.AreEqual(expectedProvider, result[0].Provider);
}
public async Task Handle(AggregateReportRecordBatch message)
{
Stopwatch stopwatch = Stopwatch.StartNew();
_log.LogInformation($"Enricher received batch of {message.Records?.Count ?? 0} aggregate reports");
List <IpAddressDetailsRequest> requests = message
.Records.Select(x => new IpAddressDetailsRequest(x.HostSourceIp, x.EffectiveDate))
.ToList();
List <IpAddressDetails> responses = await _ipAddressProcessor.Process(requests);
List <AggregateReportRecordEnriched> enrichedReports = new List <AggregateReportRecordEnriched>();
foreach (AggregateReportRecord aggregateReportRecord in message.Records)
{
IpAddressDetails ipAddressDetails = responses.FirstOrDefault(x => x.IpAddress == aggregateReportRecord.HostSourceIp);
if (ipAddressDetails is null)
{
_log.LogInformation($"Unable to enrich message for ip {aggregateReportRecord.HostSourceIp} and date {aggregateReportRecord.EffectiveDate}");
}
OrganisationalDomain organisationalDomain = await _organisationalDomainProvider.GetOrganisationalDomain(aggregateReportRecord.HeaderFrom.Trim().Trim('.').ToLower());
AggregateReportRecordEnriched aggregateReportRecordEnriched = _aggregateReportRecordEnrichedFactory.Create(ipAddressDetails, aggregateReportRecord, organisationalDomain.OrgDomain, message.CorrelationId, message.Id);
enrichedReports.Add(aggregateReportRecordEnriched);
}
foreach (AggregateReportRecordEnriched aggregateReportRecordEnriched in enrichedReports)
{
await _publisher.Publish(aggregateReportRecordEnriched, _enricherConfig.SnsTopicArn);
}
_log.LogInformation($"Enricher published batch of {enrichedReports.Count} enriched aggregate reports from request for {message.Records.Count} in {stopwatch.Elapsed.Milliseconds} ms");
stopwatch.Stop();
}
public void ConvertingToRecordShouldTallyCorrectly(DmarcResult?spfResult, DmarcResult?dkimResult, Policy?disposition, string aggregationField)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(spfResult, dkimResult, disposition, 10000);
List <DomainDateProviderRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderRecord();
Assert.AreEqual(4, result.Count);
Assert.AreEqual("hostProvider", result[0].Provider);
Assert.AreEqual("digital.ncsc.gov.uk", result[0].Domain);
Assert.AreEqual(10000, TallyAllCounts(result[0]));
Assert.AreEqual(10000, typeof(DomainDateProviderRecord).GetProperty(aggregationField)?.GetValue(result[0], null));
Assert.AreEqual("hostProvider", result[1].Provider);
Assert.AreEqual("ncsc.gov.uk", result[1].Domain);
Assert.AreEqual(10000, TallyAllCounts(result[1]));
Assert.AreEqual(10000, typeof(DomainDateProviderRecord).GetProperty(aggregationField)?.GetValue(result[0], null));
Assert.AreEqual("All Providers", result[2].Provider);
Assert.AreEqual("digital.ncsc.gov.uk", result[2].Domain);
Assert.AreEqual(10000, TallyAllCounts(result[2]));
Assert.AreEqual(10000, typeof(DomainDateProviderRecord).GetProperty(aggregationField)?.GetValue(result[1], null));
Assert.AreEqual("All Providers", result[3].Provider);
Assert.AreEqual("ncsc.gov.uk", result[3].Domain);
Assert.AreEqual(10000, TallyAllCounts(result[3]));
Assert.AreEqual(10000, typeof(DomainDateProviderRecord).GetProperty(aggregationField)?.GetValue(result[1], null));
}
public void ConvertingToRecordsShouldSplitIntoSubdomainsAndAddAllProvider()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(headerFrom: "a.b.c.d.e.gov.uk", organisationDomain: "e.gov.uk");
List <DomainDateProviderRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderRecord();
Assert.AreEqual(10, result.Count);
Assert.AreEqual("hostProvider", result[0].Provider);
Assert.AreEqual("hostProvider", result[1].Provider);
Assert.AreEqual("hostProvider", result[2].Provider);
Assert.AreEqual("hostProvider", result[3].Provider);
Assert.AreEqual("hostProvider", result[4].Provider);
Assert.AreEqual("All Providers", result[5].Provider);
Assert.AreEqual("All Providers", result[6].Provider);
Assert.AreEqual("All Providers", result[7].Provider);
Assert.AreEqual("All Providers", result[8].Provider);
Assert.AreEqual("All Providers", result[9].Provider);
Assert.AreEqual("a.b.c.d.e.gov.uk", result[0].Domain);
Assert.AreEqual("b.c.d.e.gov.uk", result[1].Domain);
Assert.AreEqual("c.d.e.gov.uk", result[2].Domain);
Assert.AreEqual("d.e.gov.uk", result[3].Domain);
Assert.AreEqual("e.gov.uk", result[4].Domain);
Assert.AreEqual("a.b.c.d.e.gov.uk", result[5].Domain);
Assert.AreEqual("b.c.d.e.gov.uk", result[6].Domain);
Assert.AreEqual("c.d.e.gov.uk", result[7].Domain);
Assert.AreEqual("d.e.gov.uk", result[8].Domain);
Assert.AreEqual("e.gov.uk", result[9].Domain);
}
public void DkimPassAndFailShouldConvertCorrectly()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched =
CreateTestRecord(new List <string>()
{
"domain1:selector1:fail", "domain1:selector2:pass", "domain2:selector1:pass"
}, count: 13);
List <DomDatProIpDkimRecord>
result = aggregateReportRecordEnriched.ToDomDatProIpDkimRecord();
Assert.AreEqual("domain1", result[0].DkimDomain);
Assert.AreEqual("selector1", result[0].DkimSelector);
Assert.AreEqual(0, result[0].DkimPass);
Assert.AreEqual(13, result[0].DkimFail);
Assert.AreEqual("domain1", result[1].DkimDomain);
Assert.AreEqual("selector2", result[1].DkimSelector);
Assert.AreEqual(13, result[1].DkimPass);
Assert.AreEqual(0, result[1].DkimFail);
Assert.AreEqual("domain2", result[2].DkimDomain);
Assert.AreEqual("selector1", result[2].DkimSelector);
Assert.AreEqual(13, result[2].DkimPass);
Assert.AreEqual(0, result[2].DkimFail);
}
public static List <DomainDateRecord> ToDomainDateRecords(this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.').ToLower() ?? aggregateReportRecord.DomainFrom.ToLower();
string orgDomain = aggregateReportRecord.OrganisationDomainFrom?.Trim().Trim('.').ToLower() ?? domain;
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
List <string> domains = new List <string> {
domain
};
while (domain != orgDomain)
{
domain = domain.Substring(domain.IndexOf('.') + 1);
domains.Add(domain);
}
;
return(domains.Select(_ => CreateDomainDateRecord(spfResult, dkimResult, disposition, id, _, date, count)).ToList());
}
public static List <DomainDateProviderRecord> ToDomainDateProviderRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
DomainDateProviderRecord record = GetDomainDateProviderRecord(aggregateReportRecord);
return(new List <DomainDateProviderRecord> {
record, record.CloneWithDifferentProvider("All Providers")
});
}
public void ConvertingToRecordShouldTallyCorrectly(DmarcResult?spfResult, DmarcResult?dkimResult, Policy?disposition, string aggregationField)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(spfResult, dkimResult, disposition, 10000);
List <DomainDateProviderIpRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderIpRecord();
Assert.AreEqual(10000, TallyAllCounts(result[0]));
Assert.AreEqual(10000, typeof(DomainDateProviderIpRecord).GetProperty(aggregationField)?.GetValue(result[0], null));
}
public void ProviderShouldBeOverridenWhenSpfandDkimFailAndOnBlocklist(DmarcResult spfResult, DmarcResult dkimResult, int blocklistCount, string expectedProvider)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(new List <string> {
"domain1:selector1:pass"
}, spfResult, dkimResult, blockListCount: blocklistCount);
var result = aggregateReportRecordEnriched.ToDomDatProIpDkimRecord();
Assert.AreEqual(expectedProvider, result[0].Provider);
}
public void ProviderShouldBeOverridenWhenSpfandDkimFailAndOnBlocklist(DmarcResult spfResult,
DmarcResult dkimResult, int blocklistCount, string expectedProvider, string originalProvider)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched =
CreateTestRecord(spfResult, dkimResult, blockListCount: blocklistCount);
EslrSaverRecord result = aggregateReportRecordEnriched.ToEslrSaverRecord();
Assert.AreEqual(expectedProvider, result.Provider);
Assert.AreEqual(originalProvider, result.OriginalProvider);
}
public void ConvertingToRecordShouldTallyCorrectly(DmarcResult?spfResult, DmarcResult?dkimResult, Policy?disposition, string aggregationField)
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(spfResult, dkimResult, disposition, 10000, "digital.ncsc.gov.uk", new DateTime(2001, 01, 01));
DomainDateRecord result = aggregateReportRecordEnriched.ToDomainDateRecord();
Assert.AreEqual(new DateTime(2001, 01, 01), result.Date);
Assert.AreEqual("digital.ncsc.gov.uk", result.Domain);
Assert.AreEqual(10000, typeof(DomainDateRecord).GetProperty(aggregationField)?.GetValue(result, null));
}
public static List <DomainDateProviderIpRecord> ToDomainDateProviderIpRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.').ToLower() ?? aggregateReportRecord.DomainFrom.ToLower();
string ip = aggregateReportRecord.HostSourceIp;
string provider = aggregateReportRecord.HostProvider;
string originalProvider = null;
string hostname = aggregateReportRecord.HostName;
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
int spfMisalignedCount = (spfResult == DmarcResult.fail && aggregateReportRecord.SpfPassCount > 0 ? count : 0);
int dkimMisAlignedCount = (dkimResult == DmarcResult.fail && aggregateReportRecord.DkimPassCount > 0 ? count : 0);
int proxyBlockListCount = aggregateReportRecord.ProxyBlockListCount;
int suspiciousNetworkBlockListCount = aggregateReportRecord.SuspiciousNetworkBlockListCount;
int hijackedNetworkBlockListCount = aggregateReportRecord.HijackedNetworkBlockListCount;
int endUserNetworkBlockListCount = aggregateReportRecord.EndUserNetworkBlockListCount;
int spamSourceBlockListCount = aggregateReportRecord.SpamSourceBlockListCount;
int malwareBlockListCount = aggregateReportRecord.MalwareBlockListCount;
int endUserBlockListCount = aggregateReportRecord.EndUserBlockListCount;
int bounceReflectorBlockListCount = aggregateReportRecord.BounceReflectorBlockListCount;
int forwarded = aggregateReportRecord.Forwarded ? count : 0;
int sampledOut = aggregateReportRecord.SampledOut ? count : 0;
int trustedForwarder = aggregateReportRecord.TrustedForwarder ? count : 0;
int mailingList = aggregateReportRecord.MailingList ? count : 0;
int localPolicy = aggregateReportRecord.LocalPolicy ? count : 0;
int arc = aggregateReportRecord.Arc ? count : 0;
int otherOverrideReason = aggregateReportRecord.OtherOverrideReason ? count : 0;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
proxyBlockListCount + suspiciousNetworkBlockListCount + hijackedNetworkBlockListCount +
endUserNetworkBlockListCount + spamSourceBlockListCount + malwareBlockListCount +
endUserBlockListCount + bounceReflectorBlockListCount > 0)
{
originalProvider = provider;
provider = "Blocklisted";
}
DomainDateProviderIpRecord record = CreateDomainDateProviderIp(spfResult, dkimResult, disposition, id, domain,
date, provider, originalProvider, ip, hostname, count, spfMisalignedCount, dkimMisAlignedCount, proxyBlockListCount, suspiciousNetworkBlockListCount,
hijackedNetworkBlockListCount, endUserNetworkBlockListCount, spamSourceBlockListCount, malwareBlockListCount,
endUserBlockListCount, bounceReflectorBlockListCount, forwarded, sampledOut, trustedForwarder,
mailingList, localPolicy, arc, otherOverrideReason);
return(new List <DomainDateProviderIpRecord> {
record, record.CloneWithDifferentProvider("All Providers")
});
}
public void BlocklistFlagsSetCorrectly(string flagName, string countField)
{
IpAddressDetails response = CreateResponse();
response.BlockListOccurrences.Add(new BlocklistAppearance(flagName, "source", "description"));
AggregateReportRecord source = CreateSource();
AggregateReportRecordEnriched result = _aggregateReportRecordEnrichedFactory.Create(response, source, "", "", "");
Assert.AreEqual(1, typeof(AggregateReportRecordEnriched).GetProperty(countField)?.GetValue(result, null));
}
public void ConvertingToRecordsShouldCreateAllProvidersRecord()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(headerFrom: "a.b.c.d.e.gov.uk", organisationDomain: "e.gov.uk");
List <DomainDateProviderRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderRecord();
Assert.AreEqual(2, result.Count);
Assert.AreEqual("hostProvider", result[0].Provider);
Assert.AreEqual("a.b.c.d.e.gov.uk", result[0].Domain);
Assert.AreEqual("All Providers", result[1].Provider);
Assert.AreEqual("a.b.c.d.e.gov.uk", result[1].Domain);
}
public void SpfFailShouldSConvertCorrectly()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched =
CreateTestRecord(new List <string>()
{
"domain11:fail"
}, count: 12);
List <DomDatProIpSpfRecord>
result = aggregateReportRecordEnriched.ToDomDatProIpSpfRecord();
Assert.AreEqual("domain11", result[0].SpfDomain);
Assert.AreEqual(0, result[0].SpfPass);
Assert.AreEqual(12, result[0].SpfFail);
}
public void ConvertingToRecordsShouldCreateAllProvidersRecord()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(hostSourceIp: "192.168.123.123", hostname: "mail.outbound.provider.com", hostProvider: "Test Provider");
List <DomainDateProviderIpRecord> result = aggregateReportRecordEnriched.ToDomainDateProviderIpRecord();
Assert.AreEqual(2, result.Count);
Assert.AreEqual("Test Provider", result[0].Provider);
Assert.AreEqual("mail.outbound.provider.com", result[0].Hostname);
Assert.AreEqual("192.168.123.123", result[0].Ip);
Assert.AreEqual("All Providers", result[1].Provider);
Assert.AreEqual("mail.outbound.provider.com", result[1].Hostname);
Assert.AreEqual("192.168.123.123", result[1].Ip);
}
public void ConvertingToRecordsShouldCreateAllProvidersRecord()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(new List <string>()
{
"domain1:selector1:pass"
});
List <DomDatProIpDkimRecord> result = aggregateReportRecordEnriched.ToDomDatProIpDkimRecord();
Assert.AreEqual(2, result.Count);
Assert.AreEqual("mail.host.provider", result[0].Provider);
Assert.AreEqual("domain1", result[0].DkimDomain);
Assert.AreEqual("All Providers", result[1].Provider);
Assert.AreEqual("domain1", result[1].DkimDomain);
}
public static List <DomainDateProviderSubdomainRecord> ToDomainDateProviderSubdomainRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.').ToLower() ?? aggregateReportRecord.DomainFrom.ToLower();
string orgDomain = aggregateReportRecord.OrganisationDomainFrom?.Trim().Trim('.').ToLower() ?? domain;
string provider = aggregateReportRecord.HostProvider;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
aggregateReportRecord.ProxyBlockListCount + aggregateReportRecord.SuspiciousNetworkBlockListCount + aggregateReportRecord.HijackedNetworkBlockListCount + aggregateReportRecord.EndUserNetworkBlockListCount + aggregateReportRecord.SpamSourceBlockListCount + aggregateReportRecord.MalwareBlockListCount + aggregateReportRecord.EndUserBlockListCount + aggregateReportRecord.BounceReflectorBlockListCount > 0)
{
provider = "Blocklisted";
}
string subdomain = domain;
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
// Don't add subdomain==domain (this data comes from MailCheck.AggregateReport.DomainDateProvider)
List <Tuple <string, string> > subdomainParentDomainPairs = new List <Tuple <string, string> > {
};
// Add {subdomain, parentDomain} where the parent is not higher than the org domain
// e.g. a.b.c.gov.uk => {a.b.c.gov.uk, b.c.gov.uk}, {b.c.gov.uk, c.gov.uk}
while (domain != orgDomain)
{
domain = domain.Substring(domain.IndexOf('.') + 1);
subdomainParentDomainPairs.Add(Tuple.Create(subdomain, domain));
subdomain = subdomain.Substring(subdomain.IndexOf('.') + 1);
}
List <DomainDateProviderSubdomainRecord> resultSets = subdomainParentDomainPairs.Select(x =>
CreateDomainDateProvider(spfResult, dkimResult, disposition, id, x.Item2, date, provider, x.Item1, count))
.ToList();
List <DomainDateProviderSubdomainRecord> allProviderResultSets = resultSets.Select(x => x.CloneWithDifferentProvider("All Providers")).ToList();
resultSets.AddRange(allProviderResultSets);
return(resultSets);
}
public static List <DomDatProIpDkimRecord> ToDomDatProIpDkimRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.').ToLower() ?? aggregateReportRecord.DomainFrom.ToLower();
string ip = aggregateReportRecord.HostSourceIp;
string provider = aggregateReportRecord.HostProvider;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
aggregateReportRecord.ProxyBlockListCount + aggregateReportRecord.SuspiciousNetworkBlockListCount + aggregateReportRecord.HijackedNetworkBlockListCount + aggregateReportRecord.EndUserNetworkBlockListCount + aggregateReportRecord.SpamSourceBlockListCount + aggregateReportRecord.MalwareBlockListCount + aggregateReportRecord.EndUserBlockListCount + aggregateReportRecord.BounceReflectorBlockListCount > 0)
{
provider = "Blocklisted";
}
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
List <string> dkimAuthResults = aggregateReportRecord.DkimAuthResults;
List <Tuple <string, string, string> > dkimDomainResults = new List <Tuple <string, string, string> > {
};
if (dkimAuthResults.Any())
{
foreach (string dkimDomainResult in dkimAuthResults)
{
string dkimDomain = dkimDomainResult.Split(':')[0];
string dkimSelector = dkimDomainResult.Split(':')[1];
string dkimResult = dkimDomainResult.Split(':')[2].ToLower();
dkimDomainResults.Add(Tuple.Create(dkimDomain, dkimSelector, dkimResult));
}
}
List <DomDatProIpDkimRecord> resultSets = dkimDomainResults.Select(x =>
CreateDomDatProIpDkim(id, domain, date, provider, ip, count, x.Item1, x.Item2, x.Item3))
.ToList();
List <DomDatProIpDkimRecord> allProviderResultSets = resultSets.Select(x => x.CloneWithDifferentProvider("All Providers")).ToList();
resultSets.AddRange(allProviderResultSets);
return(resultSets);
}
public static List <DomainDateProviderRecord> ToDomainDateProviderRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.').ToLower() ??
aggregateReportRecord.DomainFrom.ToLower();
string orgDomain = aggregateReportRecord.OrganisationDomainFrom?.Trim().Trim('.').ToLower() ?? domain;
string provider = aggregateReportRecord.HostProvider;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
aggregateReportRecord.ProxyBlockListCount + aggregateReportRecord.SuspiciousNetworkBlockListCount + aggregateReportRecord.HijackedNetworkBlockListCount + aggregateReportRecord.EndUserNetworkBlockListCount + aggregateReportRecord.SpamSourceBlockListCount + aggregateReportRecord.MalwareBlockListCount + aggregateReportRecord.EndUserBlockListCount + aggregateReportRecord.BounceReflectorBlockListCount > 0)
{
provider = "Blocklisted";
}
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
List <string> domainNames = new List <string> {
domain
};
while (domain != orgDomain)
{
domain = domain.Substring(domain.IndexOf('.') + 1);
domainNames.Add(domain);
}
List <DomainDateProviderRecord> resultSets = domainNames.Select(x =>
CreateDomainDateProvider(spfResult, dkimResult, disposition, id, x, date, provider, count))
.ToList();
List <DomainDateProviderRecord> allProviders =
resultSets.Select(_ => _.CloneWithDifferentProvider("All Providers")).ToList();
resultSets.AddRange(allProviders);
return(resultSets);
}
public void ConvertingToRecordShouldSplitIntoSubdomains()
{
AggregateReportRecordEnriched aggregateReportRecordEnriched = CreateTestRecord(headerFrom: "a.b.c.d.e.gov.uk", organisationDomain: "e.gov.uk", date: new DateTime(2001, 01, 01));
List <DomainDateRecord> result = aggregateReportRecordEnriched.ToDomainDateRecords();
Assert.AreEqual(5, result.Count);
Assert.AreEqual(new DateTime(2001, 01, 01), result[0].Date);
Assert.AreEqual(new DateTime(2001, 01, 01), result[1].Date);
Assert.AreEqual(new DateTime(2001, 01, 01), result[2].Date);
Assert.AreEqual(new DateTime(2001, 01, 01), result[3].Date);
Assert.AreEqual(new DateTime(2001, 01, 01), result[4].Date);
Assert.AreEqual("a.b.c.d.e.gov.uk", result[0].Domain);
Assert.AreEqual("b.c.d.e.gov.uk", result[1].Domain);
Assert.AreEqual("c.d.e.gov.uk", result[2].Domain);
Assert.AreEqual("d.e.gov.uk", result[3].Domain);
Assert.AreEqual("e.gov.uk", result[4].Domain);
}
public static DomainDateRecord ToDomainDateRecord(this AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom?.Trim().Trim('.') ?? aggregateReportRecord.DomainFrom;
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.none)
{
return(new DomainDateRecord(id, domain, date, count, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.none)
{
return(new DomainDateRecord(id, domain, date, 0, count, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.none)
{
return(new DomainDateRecord(id, domain, date, 0, 0, count, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.fail && disposition == Policy.none)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, count, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.quarantine)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, count, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.quarantine)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, count, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.quarantine)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, count, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.fail && disposition == Policy.quarantine)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, 0, count, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.reject)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, 0, 0, count, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.reject)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, 0, 0, 0, count, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.reject)
{
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, count, 0));
}
return(new DomainDateRecord(id, domain, date, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, count));
}
public static EslrSaverRecord ToEslrSaverRecord(
this AggregateReportRecordEnriched aggregateReportRecord)
{
long recordId = long.Parse(aggregateReportRecord.RecordId);
DateTime effectiveDate = aggregateReportRecord.EffectiveDate.Date;
string domain = DomainNameUtils.ToCanonicalDomainName(aggregateReportRecord.HeaderFrom ?? aggregateReportRecord.DomainFrom);
string reverseDomain = DomainNameUtils.ReverseDomainName(domain);
string provider = aggregateReportRecord.HostProvider;
string originalProvider = null;
string reporterOrgName = aggregateReportRecord.ReporterOrgName;
string ip = aggregateReportRecord.HostSourceIp;
int count = aggregateReportRecord.Count;
string disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none).ToString();
string dkim = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail).ToString();
string spf = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail).ToString();
string envelopeTo = aggregateReportRecord.EnvelopeTo;
string envelopeFrom = aggregateReportRecord.EnvelopeFrom;
string headerFrom = aggregateReportRecord.HeaderFrom;
string organisationDomainFrom = aggregateReportRecord.OrganisationDomainFrom;
string spfAuthResults = string.Join(',', aggregateReportRecord.SpfAuthResults);
int spfPassCount = aggregateReportRecord.SpfPassCount;
int spfFailCount = aggregateReportRecord.SpfFailCount;
string dkimAuthResults = string.Join(',', aggregateReportRecord.DkimAuthResults);
int dkimPassCount = aggregateReportRecord.DkimPassCount;
int dkimFailCount = aggregateReportRecord.DkimFailCount;
int forwarded = aggregateReportRecord.Forwarded ? count : 0;
int sampledOut = aggregateReportRecord.SampledOut ? count : 0;
int trustedForwarder = aggregateReportRecord.TrustedForwarder ? count : 0;
int mailingList = aggregateReportRecord.MailingList ? count : 0;
int localPolicy = aggregateReportRecord.LocalPolicy ? count : 0;
int arc = aggregateReportRecord.Arc ? count : 0;
int otherOverrideReason = aggregateReportRecord.OtherOverrideReason ? count : 0;
string hostName = aggregateReportRecord.HostName;
string hostOrgDomain = aggregateReportRecord.HostOrgDomain;
string hostProvider = aggregateReportRecord.HostProvider;
int hostAsNumber = aggregateReportRecord.HostAsNumber;
string hostAsDescription = aggregateReportRecord.HostAsDescription;
string hostCountry = aggregateReportRecord.HostCountry;
int proxyBlockListCount = aggregateReportRecord.ProxyBlockListCount;
int suspiciousNetworkBlockListCount = aggregateReportRecord.SuspiciousNetworkBlockListCount;
int hijackedNetworkBlockListCount = aggregateReportRecord.HijackedNetworkBlockListCount;
int endUserNetworkBlockListCount = aggregateReportRecord.EndUserNetworkBlockListCount;
int spamSourceBlockListCount = aggregateReportRecord.SpamSourceBlockListCount;
int malwareBlockListCount = aggregateReportRecord.MalwareBlockListCount;
int endUserBlockListCount = aggregateReportRecord.EndUserBlockListCount;
int bounceReflectorBlockListCount = aggregateReportRecord.BounceReflectorBlockListCount;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
proxyBlockListCount + suspiciousNetworkBlockListCount + hijackedNetworkBlockListCount +
endUserNetworkBlockListCount + spamSourceBlockListCount + malwareBlockListCount +
endUserBlockListCount + bounceReflectorBlockListCount > 0)
{
originalProvider = provider;
provider = "Blocklisted";
}
return(new EslrSaverRecord(
recordId, effectiveDate, domain, reverseDomain, provider, originalProvider, reporterOrgName, ip, count,
disposition, dkim, spf, envelopeTo, envelopeFrom, headerFrom, organisationDomainFrom,
spfAuthResults, spfPassCount, spfFailCount, dkimAuthResults, dkimPassCount, dkimFailCount,
forwarded, sampledOut, trustedForwarder, mailingList, localPolicy, arc, otherOverrideReason,
hostName, hostOrgDomain, hostProvider, hostAsNumber, hostAsDescription, hostCountry,
proxyBlockListCount, suspiciousNetworkBlockListCount, hijackedNetworkBlockListCount,
endUserNetworkBlockListCount, spamSourceBlockListCount, malwareBlockListCount,
endUserBlockListCount, bounceReflectorBlockListCount
));
}
private static DomainDateProviderRecord GetDomainDateProviderRecord(
AggregateReportRecordEnriched aggregateReportRecord)
{
long id = long.Parse(aggregateReportRecord.RecordId);
string domain = aggregateReportRecord.HeaderFrom.Trim().TrimEnd('.') ?? aggregateReportRecord.DomainFrom;
string provider = aggregateReportRecord.HostProvider;
if (aggregateReportRecord.Dkim == DmarcResult.fail &&
aggregateReportRecord.Spf == DmarcResult.fail &&
aggregateReportRecord.ProxyBlockListCount + aggregateReportRecord.SuspiciousNetworkBlockListCount + aggregateReportRecord.HijackedNetworkBlockListCount + aggregateReportRecord.EndUserNetworkBlockListCount + aggregateReportRecord.SpamSourceBlockListCount + aggregateReportRecord.MalwareBlockListCount + aggregateReportRecord.EndUserBlockListCount + aggregateReportRecord.BounceReflectorBlockListCount > 0)
{
provider = "Blocklisted";
}
DateTime date = aggregateReportRecord.EffectiveDate.Date;
int count = aggregateReportRecord.Count;
DmarcResult spfResult = aggregateReportRecord.Spf.GetValueOrDefault(DmarcResult.fail);
DmarcResult dkimResult = aggregateReportRecord.Dkim.GetValueOrDefault(DmarcResult.fail);
Policy disposition = aggregateReportRecord.Disposition.GetValueOrDefault(Policy.none);
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.none)
{
return(new DomainDateProviderRecord(id, domain, date, provider, count, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.none)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, count, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.none)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, count, 0, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.fail && disposition == Policy.none)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, count, 0, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.quarantine)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, count, 0, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.quarantine)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, count, 0, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.quarantine)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, count, 0, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.fail && disposition == Policy.quarantine)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, 0, count, 0, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.pass && disposition == Policy.reject)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, 0, 0, count, 0, 0, 0));
}
if (spfResult == DmarcResult.pass && dkimResult == DmarcResult.fail && disposition == Policy.reject)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, 0, 0, 0, count, 0, 0));
}
if (spfResult == DmarcResult.fail && dkimResult == DmarcResult.pass && disposition == Policy.reject)
{
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, count, 0));
}
return(new DomainDateProviderRecord(id, domain, date, provider, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, count));
}
