public static async Task SeedCsvAndDynamoJson(string targetFile, string jsonFile, int clientCount,
int totalCount)
{
var generator = new EntityGenerator();
File.WriteAllText(jsonFile, "");
await using var dWriter = new StreamWriter(jsonFile);
using var dynamoWriter = new JsonTextWriter(dWriter);
File.WriteAllText(targetFile, "");
await using var sWriter = new StreamWriter(targetFile);
using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture);
writer.Configuration.Delimiter = "|";
writer.WriteHeader <Entity>();
dynamoWriter.WriteStartArray();
foreach (var element in generator.GenerateWithInverseExponentialBinarySequence(clientCount, 5,
totalCount + clientCount))
{
writer.WriteRecord(element);
var next = writer.NextRecordAsync();
element.Write(dynamoWriter);
await next;
}
dynamoWriter.WriteEndArray();
dynamoWriter.Flush();
}
static void Generate()
{
var gen = new EntityGenerator();
var entities = gen.GenerateWithInverseExponentialBinarySequence(1000, 1000000, 5);
const string file = @"C:\Code\Marketplace\checkout-entities-api\performance\data\data.csv";
File.WriteAllText(file, "");
using var sWriter = new StreamWriter(file);
using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture);
writer.Configuration.HasHeaderRecord = false;
var client = new AmazonDynamoDBClient(
new BasicAWSCredentials("XX", "XX"),
new AmazonDynamoDBConfig
{
AuthenticationRegion = "eu-west-1",
ServiceURL = "http://*****:*****@"C:\Code\Marketplace\checkout-entities-api\performance\data\ids.csv";
File.WriteAllText(idFile, "");
using var idSWriter = new StreamWriter(idFile);
using var idWriter = new CsvWriter(idSWriter, CultureInfo.CurrentCulture);
idWriter.Configuration.RegisterClassMap <EntityIdMap>();
var data = new Entity[25];
var i = 0;
var tasks = new List <Task>(10);
var entitiesToRetry = new List <Dictionary <string, List <WriteRequest> > >();
foreach (var entity in entities)
{
Write(entity, idWriter);
Write(entity, writer);
data[i] = entity;
i++;
if (i != 25)
{
continue;
}
i = 0;
tasks.Add(Task.Run(() => SendToDynamo(data, client, entitiesToRetry)));
if (tasks.Count != 10)
{
continue;
}
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
using var toRetry = File.CreateText(@"C:\Code\Marketplace\checkout-entities-api\performance\data\ToRetry.csv");
var serializer = new JsonSerializer();
serializer.Serialize(toRetry, entitiesToRetry);
}
