public async Task InsertULNs(IEnumerable <long> ulns)
{
using (var logger = ESFA.DC.Logging.LoggerManager.CreateDefaultLogger())
{
var stopwatch = new Stopwatch();
var ulnHelper = new ULNHelper();
stopwatch.Start();
//get data from DB.
// var ulnData = ulnHelper.GetAllULNs();
// var ulndataInMs = stopwatch.ElapsedMilliseconds;
// stopwatch.Restart();
//add it into the reliabledictionary
var ulnsDictionary = await StateManager.GetOrAddAsync <IReliableDictionary <long, long> >("ULNsv2");
using (var tx = StateManager.CreateTransaction())
{
try
{
//clear existing values
//await ulnsDictionary.ClearAsync();
foreach (var uln in ulns)
{
await ulnsDictionary.TryAddAsync(tx, uln, uln, TimeSpan.FromMinutes(2), CancellationToken.None);
}
await tx.CommitAsync();
}
catch (Exception ex)
{
tx.Abort();
logger.LogError("Error while saving ULN into dic", ex);
}
}
stopwatch.Stop();
var dataInsertionInMs = stopwatch.ElapsedMilliseconds;
logger.LogInfo($"ULN Data records:{ulns.Count()} saved in reliabledic: {dataInsertionInMs}");
}
}
public async Task InsertULNs()
{
using (var logger = ESFA.DC.Logging.LoggerManager.CreateDefaultLogger())
{
var ulnv2Uri = new Uri("fabric:/DCT.ILR.Processing.POC/DCT.ILR.Data.ULNv2");
//clear dictionary values first
var clearULNTasks = new List <Task <bool> >();
for (int i = 0; i < 10; i++)
{
var ulnv2DataServiceProxy = _serviceProxyFactory.CreateServiceProxy <IULNv2DataService>(
ulnv2Uri,
new ServicePartitionKey(i), TargetReplicaSelector.PrimaryReplica,
"ULNdataServiceRemotingListener");
clearULNTasks.Add(ulnv2DataServiceProxy.ClearULNs());
}
await Task.WhenAll(clearULNTasks);
logger.LogInfo($"Cleared all ULNs");
var stopwatch = new Stopwatch();
var ulnHelper = new ULNHelper();
stopwatch.Start();
//get data from DB.
var ulnsData = ulnHelper.GetAllULNs();
var ulndataInMs = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//group by all the shards so that we can make single call for each shard
var shardsWithUlnsDictionary = ulnsData.GroupBy(ulnKvp => ulnKvp.Value)
.ToDictionary(t => t.Key, t => t.Select(r => r.Key).ToList());
await shardsWithUlnsDictionary.ParallelForEachAsync(async (ulnKvp) =>
//foreach (var ulnKvp in shardsWithUlnsDictionary)
{
//ResolvedServicePartition partition = await ServicePartitionResolver.GetDefault()
// .ResolveAsync(ulnv2Uri, new ServicePartitionKey(ulnKvp.Key), CancellationToken.None);
//ResolvedServiceEndpoint ep = partition.GetEndpoint();
//Insert to the correct shard based on the hash algorithm
var ulnv2DataService = _serviceProxyFactory.CreateServiceProxy <IULNv2DataService>(
ulnv2Uri,
new ServicePartitionKey(ulnKvp.Key), TargetReplicaSelector.PrimaryReplica,
"ULNdataServiceRemotingListener");
//var size = GetObjectSize(ulnKvp.Value);
var ulnsShard = ulnKvp.Value;
//if the total records count is greater then 100000 then send it in chunks
if (ulnKvp.Value.Count > 1_000_000)
{
//var tasks = new List<Task>();
var totalCount = ulnsShard.Count;
var pageSize = 100000;
var page = 1;
var skip = 0;
while (skip < totalCount)
{
await ulnv2DataService.InsertULNs(ulnsShard.Skip(skip).Take(pageSize).ToList());
page++;
skip = pageSize * (page - 1);
}
}
else
{
await ulnv2DataService.InsertULNs(ulnsShard);
}
});
var ulnSavedInMs = stopwatch.ElapsedMilliseconds;
logger.LogInfo($"Retrieved ULNs from DB in: {ulndataInMs}");
logger.LogInfo($"Inserted ULNs into SF in: {ulnSavedInMs}");
}
}