public async Task <List <IpAddressDetails> > Process(List <IpAddressDetailsRequest> requests) { _log.LogInformation($"IpAddressProcessor received request for {requests.Count} ip address details"); List <IpAddressDetails> cachedDetails = await _ipAddressDetailsDao.GetIpAddressDetails(requests); List <IpAddressDetailsRequest> cacheMisses = requests .Where(request => !cachedDetails.Any(response => response.IpAddress == request.IpAddress && response.Date == request.Date)) .ToList(); if (cacheMisses.Any()) { _log.LogInformation($"Looking up {cacheMisses.Count} IpAddressDetails not found in database"); List <IpAddressDetails> lookedUpDetails = await _ipAddressLookup.Lookup(cacheMisses); await _ipAddressDetailsDao.SaveIpAddressDetails(lookedUpDetails); cachedDetails.AddRange(lookedUpDetails); } _log.LogInformation($"IpAddressProcessor returned {cachedDetails.Count} IpAddressDetails of {requests.Count} requested"); return(cachedDetails); }
public async Task Migrate() { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); Console.WriteLine($"Getting all existing records"); List <(DateTime, ReverseDnsResult)> existingReverseDnsRecords = await GetExistingReverseDnsResults(stopwatch); Console.WriteLine($"Grouping all existing records at {stopwatch.ElapsedMilliseconds} ms"); IEnumerable <IGrouping <DateTime, ReverseDnsResult> > recordsGroupedByDate = existingReverseDnsRecords.GroupBy(x => x.Item1, y => y.Item2); double updateTally = 0; foreach (IGrouping <DateTime, ReverseDnsResult> recordGrouping in recordsGroupedByDate) { DateTime date = recordGrouping.Key; List <ReverseDnsResult> recordsToSaveForDate = recordGrouping.ToList(); List <IpAddressDetails> ipAddressDetailsForDate = new List <IpAddressDetails>(); foreach (ReverseDnsResult reverseDnsResult in recordsToSaveForDate) { IpAddressDetails ipAddressDetails = new IpAddressDetails(reverseDnsResult.OriginalIpAddress, date, null, null, null, null, reverseDnsResult.ForwardResponses, null, null, date); ipAddressDetailsForDate.Add(ipAddressDetails); } List <Task> insertsToDo = ipAddressDetailsForDate.Batch(1000).Select(batch => _ipAddressDetailsDao.SaveIpAddressDetails(batch.ToList())).ToList(); await Task.WhenAll(insertsToDo); updateTally += ipAddressDetailsForDate.Count; double percentDone = updateTally / 4500000d; Console.Write($"\rUpdated {updateTally} = {percentDone:P2}% after {stopwatch.Elapsed.Minutes} mins {stopwatch.Elapsed.Seconds} secs"); } Console.WriteLine($"\nDone updating at {stopwatch.ElapsedMilliseconds} ms"); Console.WriteLine("Press a key"); Console.ReadLine(); }