public static async Task SeedCsvAndDynamoJson(string targetFile, string jsonFile, int clientCount,
int totalCount)
{
var generator = new EntityGenerator();
File.WriteAllText(jsonFile, "");
await using var dWriter = new StreamWriter(jsonFile);
using var dynamoWriter = new JsonTextWriter(dWriter);
File.WriteAllText(targetFile, "");
await using var sWriter = new StreamWriter(targetFile);
using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture);
writer.Configuration.Delimiter = "|";
writer.WriteHeader <Entity>();
dynamoWriter.WriteStartArray();
foreach (var element in generator.GenerateWithInverseExponentialBinarySequence(clientCount, 5,
totalCount + clientCount))
{
writer.WriteRecord(element);
var next = writer.NextRecordAsync();
element.Write(dynamoWriter);
await next;
}
dynamoWriter.WriteEndArray();
dynamoWriter.Flush();
}
public static async Task SeedEntities()
{
var generator = new EntityGenerator();
const string readEntity = @"C:\Temp\checkout-entities-api\data1\ReadEntityById.csv";
const string readClient = @"C:\Temp\checkout-entities-api\data1\ReadEntitiesByClient.csv";
const string create = @"C:\Temp\checkout-entities-api\data1\CreateEntity.csv";
File.WriteAllText(readEntity, "");
File.WriteAllText(create, "");
File.WriteAllText(readClient, "");
using var entityWriter = new CsvWriter(new StreamWriter(readEntity), CultureInfo.CurrentCulture);
entityWriter.Configuration.Delimiter = "|";
entityWriter.WriteHeader <Entity>();
entityWriter.NextRecord();
using var createWriter = new CsvWriter(new StreamWriter(create), CultureInfo.CurrentCulture);
createWriter.Configuration.Delimiter = "|";
createWriter.Configuration.RegisterClassMap <EntityMap>();
createWriter.WriteHeader <Entity>();
createWriter.NextRecord();
using var writer = new CsvWriter(new StreamWriter(readClient), CultureInfo.CurrentCulture);
writer.Configuration.ShouldQuote = (entry, context) => false;
writer.Configuration.Delimiter = "|";
writer.WriteHeader <Client>();
writer.NextRecord();
const int totalCount = 50;
const int clientCount = totalCount / 20;
const int total = totalCount + clientCount;
var rnd = new Random();
var horizontalDist = Enumerable.Range(0, clientCount).Select(s => rnd.NextDouble()).ToArray();
var verticalDistribution = Enumerable.Range(0, 5)
.Select(s => Math.Pow(2, s * -1))
.ToArray();
var lastClient = "";
var entitiesInClient = new List <string>();
foreach (var element in generator.Generate(clientCount, 5, total, verticalDistribution, horizontalDist,
false))
{
createWriter.WriteRecord(element);
createWriter.NextRecord();
entityWriter.WriteRecord(element);
entityWriter.NextRecord();
if (lastClient == element.ClientId)
{
entitiesInClient.Add(element.EntityId);
continue;
}
if (lastClient == "")
{
entitiesInClient.Add(element.EntityId);
lastClient = element.ClientId;
continue;
}
var client = new Client
{
ClientId = lastClient,
EntityCount = entitiesInClient.Count,
Entities = string.Join(",", entitiesInClient)
};
writer.WriteRecord(client);
writer.NextRecord();
entitiesInClient.Clear();
entitiesInClient.Add(element.EntityId);
lastClient = element.ClientId;
}
}
static void Generate()
{
var gen = new EntityGenerator();
var entities = gen.GenerateWithInverseExponentialBinarySequence(1000, 1000000, 5);
const string file = @"C:\Code\Marketplace\checkout-entities-api\performance\data\data.csv";
File.WriteAllText(file, "");
using var sWriter = new StreamWriter(file);
using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture);
writer.Configuration.HasHeaderRecord = false;
var client = new AmazonDynamoDBClient(
new BasicAWSCredentials("XX", "XX"),
new AmazonDynamoDBConfig
{
AuthenticationRegion = "eu-west-1",
ServiceURL = "http://*****:*****@"C:\Code\Marketplace\checkout-entities-api\performance\data\ids.csv";
File.WriteAllText(idFile, "");
using var idSWriter = new StreamWriter(idFile);
using var idWriter = new CsvWriter(idSWriter, CultureInfo.CurrentCulture);
idWriter.Configuration.RegisterClassMap <EntityIdMap>();
var data = new Entity[25];
var i = 0;
var tasks = new List <Task>(10);
var entitiesToRetry = new List <Dictionary <string, List <WriteRequest> > >();
foreach (var entity in entities)
{
Write(entity, idWriter);
Write(entity, writer);
data[i] = entity;
i++;
if (i != 25)
{
continue;
}
i = 0;
tasks.Add(Task.Run(() => SendToDynamo(data, client, entitiesToRetry)));
if (tasks.Count != 10)
{
continue;
}
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
using var toRetry = File.CreateText(@"C:\Code\Marketplace\checkout-entities-api\performance\data\ToRetry.csv");
var serializer = new JsonSerializer();
serializer.Serialize(toRetry, entitiesToRetry);
}
