public DynamoDbMetastoreImplTest(DynamoDBContainerFixture dynamoDbContainerFixture) { dynamoDbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration(dynamoDbContainerFixture.ServiceUrl, "us-west-2") .Build(); amazonDynamoDbClient = dynamoDbMetastoreImpl.DbClient; CreateTableSchema(amazonDynamoDbClient, dynamoDbMetastoreImpl.TableName); table = Table.LoadTable(amazonDynamoDbClient, dynamoDbMetastoreImpl.TableName); JObject jObject = JObject.FromObject(keyRecord); Document document = new Document { [PartitionKey] = TestKey, [SortKey] = created.ToUnixTimeSeconds(), [AttributeKeyRecord] = Document.FromJson(jObject.ToString()), }; table.PutItemAsync(document).Wait(); document = new Document { [PartitionKey] = TestKeyWithRegionSuffix, [SortKey] = created.ToUnixTimeSeconds(), [AttributeKeyRecord] = Document.FromJson(jObject.ToString()), }; table.PutItemAsync(document).Wait(); }
private void TestBuilderPathWithEndPointConfiguration() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration("http://localhost:" + DynamoDbPort, Region) .Build(); Assert.NotNull(dbMetastoreImpl); }
private void TestBuilderPathWithRegion() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithRegion("us-west-1") .Build(); Assert.NotNull(dbMetastoreImpl); }
private void TestBuilderPathWithEndPointConfiguration() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration($"http://{hostName}:{DynamoDbPort}", Region) .Build(); Assert.NotNull(dbMetastoreImpl); }
private void TestBuilderPathWithoutKeySuffix() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration("http://localhost:" + DynamoDbPort, Region) .Build(); Assert.NotNull(dbMetastoreImpl); Assert.Equal(string.Empty, dbMetastoreImpl.GetKeySuffix()); }
private void TestPrimaryBuilderPath() { // Hack to inject default region since we don't explicitly require one be specified as we do in KMS impl AWSConfigs.AWSRegion = "us-west-2"; DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .Build(); Assert.NotNull(dbMetastoreImpl); }
private void TestStoreWithSuffixSuccess() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration("http://localhost:" + DynamoDbPort, Region) .WithKeySuffix() .Build(); bool actualValue = dbMetastoreImpl.Store(TestKey, DateTimeOffset.Now, JObject.FromObject(keyRecord)); Assert.True(actualValue); }
private void TestBuilderPathWithKeySuffix() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithKeySuffix() .Build(); Assert.NotNull(dbMetastoreImpl); Assert.True(dbMetastoreImpl.HasKeySuffix); Assert.False(dynamoDbMetastoreImpl.HasKeySuffix); }
private void TestLoadLatestWithSingleRecordAndSuffix() { DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration("http://localhost:" + DynamoDbPort, Region) .WithKeySuffix() .Build(); Option <JObject> actualJsonObject = dbMetastoreImpl.LoadLatest(TestKey); Assert.True(actualJsonObject.IsSome); Assert.True(JToken.DeepEquals(JObject.FromObject(keyRecord), (JObject)actualJsonObject)); }
private void TestPrimaryBuilderPath() { Mock <Builder> builder = new Mock <Builder>(Region); Table loadTable = Table.LoadTable(amazonDynamoDbClient, "EncryptionKey"); builder.Setup(x => x.LoadTable(It.IsAny <IAmazonDynamoDB>(), Region)) .Returns(loadTable); DynamoDbMetastoreImpl dbMetastoreImpl = builder.Object .Build(); Assert.NotNull(dbMetastoreImpl); }
private SessionFactory GetSessionFactory(bool withKeySuffix, string region) { DynamoDbMetastoreImpl.IBuildStep builder = DynamoDbMetastoreImpl.NewBuilder(region) .WithEndPointConfiguration($"http://{hostName}:{DynamoDbPort}", DefaultRegion); if (withKeySuffix) { builder = builder.WithKeySuffix(); } DynamoDbMetastoreImpl dynamoDbMetastore = builder.Build(); return(SessionFactoryGenerator.CreateDefaultSessionFactory(configFixture.KeyManagementService, dynamoDbMetastore)); }
public DynamoDbMetastoreImplTest(DynamoDBContainerFixture dynamoDbContainerFixture) { AmazonDynamoDBConfig clientConfig = new AmazonDynamoDBConfig { ServiceURL = dynamoDbContainerFixture.ServiceUrl }; amazonDynamoDbClient = new AmazonDynamoDBClient(clientConfig); CreateTableRequest request = new CreateTableRequest { TableName = TableName, AttributeDefinitions = new List <AttributeDefinition> { new AttributeDefinition(PartitionKey, ScalarAttributeType.S), new AttributeDefinition(SortKey, ScalarAttributeType.N) }, KeySchema = new List <KeySchemaElement> { new KeySchemaElement(PartitionKey, KeyType.HASH), new KeySchemaElement(SortKey, KeyType.RANGE) }, ProvisionedThroughput = new ProvisionedThroughput(1L, 1L) }; CreateTableResponse createTableResponse = amazonDynamoDbClient.CreateTableAsync(request).Result; table = Table.LoadTable(amazonDynamoDbClient, TableName); JObject jObject = JObject.FromObject(keyRecord); Document document = new Document { [PartitionKey] = TestKey, [SortKey] = created.ToUnixTimeSeconds(), [AttributeKeyRecord] = Document.FromJson(jObject.ToString()) }; Document result = table.PutItemAsync(document).Result; dynamoDbMetastoreImpl = new DynamoDbMetastoreImpl(amazonDynamoDbClient); }
public DynamoDbGlobalTableTest(DynamoDBContainerFixture dynamoDbContainerFixture, ConfigFixture configFixture) { this.configFixture = configFixture; // Use AWS SDK to create client and initialize table AmazonDynamoDBConfig amazonDynamoDbConfig = new AmazonDynamoDBConfig { ServiceURL = dynamoDbContainerFixture.ServiceUrl, AuthenticationRegion = "us-west-2", }; tempDynamoDbClient = new AmazonDynamoDBClient(amazonDynamoDbConfig); CreateTableRequest request = new CreateTableRequest { TableName = DefaultTableName, AttributeDefinitions = new List <AttributeDefinition> { new AttributeDefinition(PartitionKey, ScalarAttributeType.S), new AttributeDefinition(SortKey, ScalarAttributeType.N), }, KeySchema = new List <KeySchemaElement> { new KeySchemaElement(PartitionKey, KeyType.HASH), new KeySchemaElement(SortKey, KeyType.RANGE), }, ProvisionedThroughput = new ProvisionedThroughput(1L, 1L), }; tempDynamoDbClient.CreateTableAsync(request).Wait(); // Use a builder without the suffix dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.NewBuilder("us-west-2") .WithEndPointConfiguration(dynamoDbContainerFixture.ServiceUrl, "us-west-2") .Build(); // Connect to the same metastore but initialize it with a key suffix dynamoDbMetastoreImplWithKeySuffix = DynamoDbMetastoreImpl.NewBuilder("us-west-2") .WithEndPointConfiguration(dynamoDbContainerFixture.ServiceUrl, "us-west-2") .WithKeySuffix() .Build(); }
private IMetastore <JObject> CreateMetastore() { if (MetastoreType.Equals(MetastoreAdo, StringComparison.InvariantCultureIgnoreCase)) { string metastoreAdoConnectionString = GetParam(MetastoreAdoConnectionString); if (string.IsNullOrWhiteSpace(metastoreAdoConnectionString)) { throw new AppEncryptionException("Missing ADO connection string"); } return(AdoMetastoreImpl .NewBuilder(MySqlClientFactory.Instance, metastoreAdoConnectionString) .Build()); } if (MetastoreType.Equals(MetastoreDynamoDb, StringComparison.InvariantCultureIgnoreCase)) { return(DynamoDbMetastoreImpl.NewBuilder("us-west-2").Build()); } return(new InMemoryMetastoreImpl <JObject>()); }
private void TestBuilderPathWithTableName() { const string tempTableName = "DummyTable"; // Use AWS SDK to create client AmazonDynamoDBConfig amazonDynamoDbConfig = new AmazonDynamoDBConfig { ServiceURL = "http://localhost:8000", AuthenticationRegion = "us-west-2", }; AmazonDynamoDBClient tempDynamoDbClient = new AmazonDynamoDBClient(amazonDynamoDbConfig); CreateTableSchema(tempDynamoDbClient, tempTableName); // Put the object in temp table Table tempTable = Table.LoadTable(tempDynamoDbClient, tempTableName); JObject jObject = JObject.FromObject(keyRecord); Document document = new Document { [PartitionKey] = TestKey, [SortKey] = created.ToUnixTimeSeconds(), [AttributeKeyRecord] = Document.FromJson(jObject.ToString()), }; tempTable.PutItemAsync(document).Wait(); // Create a metastore object using the withTableName step DynamoDbMetastoreImpl dbMetastoreImpl = NewBuilder(Region) .WithEndPointConfiguration("http://localhost:" + DynamoDbPort, "us-west-2") .WithTableName(tempTableName) .Build(); Option <JObject> actualJsonObject = dbMetastoreImpl.Load(TestKey, created); // Verify that we were able to load and successfully decrypt the item from the metastore object created withTableName Assert.True(actualJsonObject.IsSome); Assert.True(JToken.DeepEquals(JObject.FromObject(keyRecord), (JObject)actualJsonObject)); }
private static async void App(Options options) { IMetastore <JObject> metastore = null; KeyManagementService keyManagementService = null; if (options.Metastore == Metastore.ADO) { if (options.AdoConnectionString != null) { logger.LogInformation("using ADO-based metastore..."); metastore = AdoMetastoreImpl .NewBuilder(MySqlClientFactory.Instance, options.AdoConnectionString) .Build(); } else { logger.LogError("ADO connection string is a mandatory parameter with Metastore Type: ADO"); Console.WriteLine(HelpText.AutoBuild(cmdOptions, null, null)); return; } } else if (options.Metastore == Metastore.DYNAMODB) { logger.LogInformation("using DynamoDB-based metastore..."); AWSConfigs.AWSRegion = "us-west-2"; metastore = DynamoDbMetastoreImpl.NewBuilder().Build(); } else { logger.LogInformation("using in-memory metastore..."); metastore = new InMemoryMetastoreImpl <JObject>(); } if (options.Kms == Kms.AWS) { if (options.PreferredRegion != null && options.RegionToArnTuples != null) { Dictionary <string, string> regionToArnDictionary = new Dictionary <string, string>(); foreach (string regionArnPair in options.RegionToArnTuples) { string[] regionArnArray = regionArnPair.Split("="); regionToArnDictionary.Add(regionArnArray[0], regionArnArray[1]); } logger.LogInformation("using AWS KMS..."); keyManagementService = AwsKeyManagementServiceImpl .NewBuilder(regionToArnDictionary, options.PreferredRegion).Build(); } else { logger.LogError("Preferred region and <region>=<arn> tuples are mandatory with KMS Type: AWS"); Console.WriteLine(HelpText.AutoBuild(cmdOptions, null, null)); return; } } else { logger.LogInformation("using static KMS..."); keyManagementService = new StaticKeyManagementServiceImpl("mysupersecretstaticmasterkey!!!!"); } CryptoPolicy cryptoPolicy = BasicExpiringCryptoPolicy .NewBuilder() .WithKeyExpirationDays(KeyExpirationDays) .WithRevokeCheckMinutes(CacheCheckMinutes) .Build(); // Setup metrics reporters and always include console. IMetricsBuilder metricsBuilder = new MetricsBuilder() .Report.ToConsole(consoleOptions => consoleOptions.FlushInterval = TimeSpan.FromSeconds(60)); // CloudWatch metrics generation if (options.EnableCloudWatch) { // Fill in when we open source our App.Metrics cloudwatch reporter separately } IMetrics metrics = metricsBuilder.Build(); // Create a session factory for this app. Normally this would be done upon app startup and the // same factory would be used anytime a new session is needed for a partition (e.g., shopper). // We've split it out into multiple using blocks to underscore this point. using (SessionFactory sessionFactory = SessionFactory .NewBuilder("productId", "reference_app") .WithMetastore(metastore) .WithCryptoPolicy(cryptoPolicy) .WithKeyManagementService(keyManagementService) .WithMetrics(metrics) .Build()) { // Now create an actual session for a partition (which in our case is a pretend shopper id). This session is used // for a transaction and is disposed automatically after use due to the IDisposable implementation. using (Session <byte[], byte[]> sessionBytes = sessionFactory.GetSessionBytes("shopper123")) { const string originalPayloadString = "mysupersecretpayload"; foreach (int i in Enumerable.Range(0, options.Iterations)) { string dataRowString; // If we get a DRR as a command line argument, we want to directly decrypt it if (options.Drr != null) { dataRowString = options.Drr; } else { // Encrypt the payload byte[] dataRowRecordBytes = sessionBytes.Encrypt(Encoding.UTF8.GetBytes(originalPayloadString)); // Consider this us "persisting" the DRR dataRowString = Convert.ToBase64String(dataRowRecordBytes); } logger.LogInformation("dataRowRecord as string = {dataRow}", dataRowString); byte[] newDataRowRecordBytes = Convert.FromBase64String(dataRowString); // Decrypt the payload string decryptedPayloadString = Encoding.UTF8.GetString(sessionBytes.Decrypt(newDataRowRecordBytes)); logger.LogInformation("decryptedPayloadString = {payload}", decryptedPayloadString); logger.LogInformation("matches = {result}", originalPayloadString.Equals(decryptedPayloadString)); } } } // Force final publish of metrics await Task.WhenAll(((IMetricsRoot)metrics).ReportRunner.RunAllAsync()); }