public static async Task SeedCsvAndDynamoJson(string targetFile, string jsonFile, int clientCount, int totalCount) { var generator = new EntityGenerator(); File.WriteAllText(jsonFile, ""); await using var dWriter = new StreamWriter(jsonFile); using var dynamoWriter = new JsonTextWriter(dWriter); File.WriteAllText(targetFile, ""); await using var sWriter = new StreamWriter(targetFile); using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture); writer.Configuration.Delimiter = "|"; writer.WriteHeader <Entity>(); dynamoWriter.WriteStartArray(); foreach (var element in generator.GenerateWithInverseExponentialBinarySequence(clientCount, 5, totalCount + clientCount)) { writer.WriteRecord(element); var next = writer.NextRecordAsync(); element.Write(dynamoWriter); await next; } dynamoWriter.WriteEndArray(); dynamoWriter.Flush(); }
public static async Task SeedEntities() { var generator = new EntityGenerator(); const string readEntity = @"C:\Temp\checkout-entities-api\data1\ReadEntityById.csv"; const string readClient = @"C:\Temp\checkout-entities-api\data1\ReadEntitiesByClient.csv"; const string create = @"C:\Temp\checkout-entities-api\data1\CreateEntity.csv"; File.WriteAllText(readEntity, ""); File.WriteAllText(create, ""); File.WriteAllText(readClient, ""); using var entityWriter = new CsvWriter(new StreamWriter(readEntity), CultureInfo.CurrentCulture); entityWriter.Configuration.Delimiter = "|"; entityWriter.WriteHeader <Entity>(); entityWriter.NextRecord(); using var createWriter = new CsvWriter(new StreamWriter(create), CultureInfo.CurrentCulture); createWriter.Configuration.Delimiter = "|"; createWriter.Configuration.RegisterClassMap <EntityMap>(); createWriter.WriteHeader <Entity>(); createWriter.NextRecord(); using var writer = new CsvWriter(new StreamWriter(readClient), CultureInfo.CurrentCulture); writer.Configuration.ShouldQuote = (entry, context) => false; writer.Configuration.Delimiter = "|"; writer.WriteHeader <Client>(); writer.NextRecord(); const int totalCount = 50; const int clientCount = totalCount / 20; const int total = totalCount + clientCount; var rnd = new Random(); var horizontalDist = Enumerable.Range(0, clientCount).Select(s => rnd.NextDouble()).ToArray(); var verticalDistribution = Enumerable.Range(0, 5) .Select(s => Math.Pow(2, s * -1)) .ToArray(); var lastClient = ""; var entitiesInClient = new List <string>(); foreach (var element in generator.Generate(clientCount, 5, total, verticalDistribution, horizontalDist, false)) { createWriter.WriteRecord(element); createWriter.NextRecord(); entityWriter.WriteRecord(element); entityWriter.NextRecord(); if (lastClient == element.ClientId) { entitiesInClient.Add(element.EntityId); continue; } if (lastClient == "") { entitiesInClient.Add(element.EntityId); lastClient = element.ClientId; continue; } var client = new Client { ClientId = lastClient, EntityCount = entitiesInClient.Count, Entities = string.Join(",", entitiesInClient) }; writer.WriteRecord(client); writer.NextRecord(); entitiesInClient.Clear(); entitiesInClient.Add(element.EntityId); lastClient = element.ClientId; } }
static void Generate() { var gen = new EntityGenerator(); var entities = gen.GenerateWithInverseExponentialBinarySequence(1000, 1000000, 5); const string file = @"C:\Code\Marketplace\checkout-entities-api\performance\data\data.csv"; File.WriteAllText(file, ""); using var sWriter = new StreamWriter(file); using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture); writer.Configuration.HasHeaderRecord = false; var client = new AmazonDynamoDBClient( new BasicAWSCredentials("XX", "XX"), new AmazonDynamoDBConfig { AuthenticationRegion = "eu-west-1", ServiceURL = "http://*****:*****@"C:\Code\Marketplace\checkout-entities-api\performance\data\ids.csv"; File.WriteAllText(idFile, ""); using var idSWriter = new StreamWriter(idFile); using var idWriter = new CsvWriter(idSWriter, CultureInfo.CurrentCulture); idWriter.Configuration.RegisterClassMap <EntityIdMap>(); var data = new Entity[25]; var i = 0; var tasks = new List <Task>(10); var entitiesToRetry = new List <Dictionary <string, List <WriteRequest> > >(); foreach (var entity in entities) { Write(entity, idWriter); Write(entity, writer); data[i] = entity; i++; if (i != 25) { continue; } i = 0; tasks.Add(Task.Run(() => SendToDynamo(data, client, entitiesToRetry))); if (tasks.Count != 10) { continue; } Task.WaitAll(tasks.ToArray()); tasks.Clear(); } using var toRetry = File.CreateText(@"C:\Code\Marketplace\checkout-entities-api\performance\data\ToRetry.csv"); var serializer = new JsonSerializer(); serializer.Serialize(toRetry, entitiesToRetry); }