public static async Task SeedCsvAndDynamoJson(string targetFile, string jsonFile, int clientCount, int totalCount) { var generator = new EntityGenerator(); File.WriteAllText(jsonFile, ""); await using var dWriter = new StreamWriter(jsonFile); using var dynamoWriter = new JsonTextWriter(dWriter); File.WriteAllText(targetFile, ""); await using var sWriter = new StreamWriter(targetFile); using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture); writer.Configuration.Delimiter = "|"; writer.WriteHeader <Entity>(); dynamoWriter.WriteStartArray(); foreach (var element in generator.GenerateWithInverseExponentialBinarySequence(clientCount, 5, totalCount + clientCount)) { writer.WriteRecord(element); var next = writer.NextRecordAsync(); element.Write(dynamoWriter); await next; } dynamoWriter.WriteEndArray(); dynamoWriter.Flush(); }
static void Generate() { var gen = new EntityGenerator(); var entities = gen.GenerateWithInverseExponentialBinarySequence(1000, 1000000, 5); const string file = @"C:\Code\Marketplace\checkout-entities-api\performance\data\data.csv"; File.WriteAllText(file, ""); using var sWriter = new StreamWriter(file); using var writer = new CsvWriter(sWriter, CultureInfo.CurrentCulture); writer.Configuration.HasHeaderRecord = false; var client = new AmazonDynamoDBClient( new BasicAWSCredentials("XX", "XX"), new AmazonDynamoDBConfig { AuthenticationRegion = "eu-west-1", ServiceURL = "http://*****:*****@"C:\Code\Marketplace\checkout-entities-api\performance\data\ids.csv"; File.WriteAllText(idFile, ""); using var idSWriter = new StreamWriter(idFile); using var idWriter = new CsvWriter(idSWriter, CultureInfo.CurrentCulture); idWriter.Configuration.RegisterClassMap <EntityIdMap>(); var data = new Entity[25]; var i = 0; var tasks = new List <Task>(10); var entitiesToRetry = new List <Dictionary <string, List <WriteRequest> > >(); foreach (var entity in entities) { Write(entity, idWriter); Write(entity, writer); data[i] = entity; i++; if (i != 25) { continue; } i = 0; tasks.Add(Task.Run(() => SendToDynamo(data, client, entitiesToRetry))); if (tasks.Count != 10) { continue; } Task.WaitAll(tasks.ToArray()); tasks.Clear(); } using var toRetry = File.CreateText(@"C:\Code\Marketplace\checkout-entities-api\performance\data\ToRetry.csv"); var serializer = new JsonSerializer(); serializer.Serialize(toRetry, entitiesToRetry); }