public async Task <List <IpAddressDetails> > Process(List <IpAddressDetailsRequest> requests)
{
_log.LogInformation($"IpAddressProcessor received request for {requests.Count} ip address details");
List <IpAddressDetails> cachedDetails = await _ipAddressDetailsDao.GetIpAddressDetails(requests);
List <IpAddressDetailsRequest> cacheMisses = requests
.Where(request => !cachedDetails.Any(response => response.IpAddress == request.IpAddress && response.Date == request.Date))
.ToList();
if (cacheMisses.Any())
{
_log.LogInformation($"Looking up {cacheMisses.Count} IpAddressDetails not found in database");
List <IpAddressDetails> lookedUpDetails = await _ipAddressLookup.Lookup(cacheMisses);
await _ipAddressDetailsDao.SaveIpAddressDetails(lookedUpDetails);
cachedDetails.AddRange(lookedUpDetails);
}
_log.LogInformation($"IpAddressProcessor returned {cachedDetails.Count} IpAddressDetails of {requests.Count} requested");
return(cachedDetails);
}
public async Task Migrate()
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Console.WriteLine($"Getting all existing records");
List <(DateTime, ReverseDnsResult)> existingReverseDnsRecords = await GetExistingReverseDnsResults(stopwatch);
Console.WriteLine($"Grouping all existing records at {stopwatch.ElapsedMilliseconds} ms");
IEnumerable <IGrouping <DateTime, ReverseDnsResult> > recordsGroupedByDate = existingReverseDnsRecords.GroupBy(x => x.Item1, y => y.Item2);
double updateTally = 0;
foreach (IGrouping <DateTime, ReverseDnsResult> recordGrouping in recordsGroupedByDate)
{
DateTime date = recordGrouping.Key;
List <ReverseDnsResult> recordsToSaveForDate = recordGrouping.ToList();
List <IpAddressDetails> ipAddressDetailsForDate = new List <IpAddressDetails>();
foreach (ReverseDnsResult reverseDnsResult in recordsToSaveForDate)
{
IpAddressDetails ipAddressDetails = new IpAddressDetails(reverseDnsResult.OriginalIpAddress, date, null, null, null, null, reverseDnsResult.ForwardResponses, null, null, date);
ipAddressDetailsForDate.Add(ipAddressDetails);
}
List <Task> insertsToDo = ipAddressDetailsForDate.Batch(1000).Select(batch => _ipAddressDetailsDao.SaveIpAddressDetails(batch.ToList())).ToList();
await Task.WhenAll(insertsToDo);
updateTally += ipAddressDetailsForDate.Count;
double percentDone = updateTally / 4500000d;
Console.Write($"\rUpdated {updateTally} = {percentDone:P2}% after {stopwatch.Elapsed.Minutes} mins {stopwatch.Elapsed.Seconds} secs");
}
Console.WriteLine($"\nDone updating at {stopwatch.ElapsedMilliseconds} ms");
Console.WriteLine("Press a key");
Console.ReadLine();
}