Exemple #1
0
        public void GetDatabaseResourceNames()
        {
            moq::Mock <FirestoreAdmin.FirestoreAdminClient> mockGrpcClient = new moq::Mock <FirestoreAdmin.FirestoreAdminClient>(moq::MockBehavior.Strict);

            mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
            GetDatabaseRequest request = new GetDatabaseRequest
            {
                DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
            };
            Database expectedResponse = new Database
            {
                DatabaseName    = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
                LocationId      = "location_iddaa574e2",
                Type            = Database.Types.DatabaseType.Unspecified,
                ConcurrencyMode = Database.Types.ConcurrencyMode.OptimisticWithEntityGroups,
                Etag            = "etage8ad7218",
            };

            mockGrpcClient.Setup(x => x.GetDatabase(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(expectedResponse);
            FirestoreAdminClient client   = new FirestoreAdminClientImpl(mockGrpcClient.Object, null);
            Database             response = client.GetDatabase(request.DatabaseName);

            xunit::Assert.Same(expectedResponse, response);
            mockGrpcClient.VerifyAll();
        }
Exemple #2
0
        public virtual async Task <GetDatabaseResponse> GetAsync(GetDatabaseRequest request)
        {
            var httpRequest = GetHttpRequestFactory.Create(request);

            using (var httpResponse = await SendAsync(httpRequest).ForAwait())
                return(await GetDatabaseResponseFactory.CreateAsync(httpResponse).ForAwait());
        }
Exemple #3
0
        public async stt::Task GetDatabaseResourceNamesAsync()
        {
            moq::Mock <FirestoreAdmin.FirestoreAdminClient> mockGrpcClient = new moq::Mock <FirestoreAdmin.FirestoreAdminClient>(moq::MockBehavior.Strict);

            mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
            GetDatabaseRequest request = new GetDatabaseRequest
            {
                DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
            };
            Database expectedResponse = new Database
            {
                DatabaseName    = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
                LocationId      = "location_iddaa574e2",
                Type            = Database.Types.DatabaseType.Unspecified,
                ConcurrencyMode = Database.Types.ConcurrencyMode.OptimisticWithEntityGroups,
                Etag            = "etage8ad7218",
            };

            mockGrpcClient.Setup(x => x.GetDatabaseAsync(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(new grpccore::AsyncUnaryCall <Database>(stt::Task.FromResult(expectedResponse), null, null, null, null));
            FirestoreAdminClient client = new FirestoreAdminClientImpl(mockGrpcClient.Object, null);
            Database             responseCallSettings = await client.GetDatabaseAsync(request.DatabaseName, gaxgrpc::CallSettings.FromCancellationToken(st::CancellationToken.None));

            xunit::Assert.Same(expectedResponse, responseCallSettings);
            Database responseCancellationToken = await client.GetDatabaseAsync(request.DatabaseName, st::CancellationToken.None);

            xunit::Assert.Same(expectedResponse, responseCancellationToken);
            mockGrpcClient.VerifyAll();
        }
Exemple #4
0
        public virtual HttpRequest Create(GetDatabaseRequest request)
        {
            Ensure.Any.IsNotNull(request, nameof(request));

            return(new HttpRequest(HttpMethod.Get, GenerateRelativeUrl(request))
                   .SetRequestTypeHeader(request.GetType()));
        }
Exemple #5
0
        public virtual async Task <GetDatabaseResponse> GetAsync(CancellationToken cancellationToken = default)
        {
            var request     = new GetDatabaseRequest(Connection.DbName);
            var httpRequest = GetHttpRequestFactory.Create(request);

            using (var httpResponse = await SendAsync(httpRequest, cancellationToken).ForAwait())
                return(await GetDatabaseResponseFactory.CreateAsync(httpResponse).ForAwait());
        }
Exemple #6
0
        public virtual async Task <GetDatabaseResponse> GetAsync(GetDatabaseRequest request)
        {
            var httpRequest = CreateHttpRequest(request);

            using (var res = await SendAsync(httpRequest).ForAwait())
            {
                return(ProcessGetDatabaseResponse(res));
            }
        }
 /// <summary>Snippet for GetDatabase</summary>
 public void GetDatabase_RequestObject()
 {
     // Snippet: GetDatabase(GetDatabaseRequest,CallSettings)
     // Create client
     DatabaseAdminClient databaseAdminClient = DatabaseAdminClient.Create();
     // Initialize request argument(s)
     GetDatabaseRequest request = new GetDatabaseRequest
     {
         DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
     };
     // Make the request
     Database response = databaseAdminClient.GetDatabase(request);
     // End snippet
 }
Exemple #8
0
        public async Task GetDatabaseAsync_RequestObject()
        {
            // Snippet: GetDatabaseAsync(GetDatabaseRequest,CallSettings)
            // Create client
            DatabaseAdminClient databaseAdminClient = await DatabaseAdminClient.CreateAsync();

            // Initialize request argument(s)
            GetDatabaseRequest request = new GetDatabaseRequest
            {
                Name = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]").ToString(),
            };
            // Make the request
            Database response = await databaseAdminClient.GetDatabaseAsync(request);

            // End snippet
        }
Exemple #9
0
        /// <summary>
        /// Gets information about a specific database.
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        public async Task <GetDatabaseResponse> GetDatabase(GetDatabaseRequest request)
        {
            var uri = new Uri($"{GetEndPoint(DatabaseServices.Databases, this.Region)}/{request.DatabaseId}");

            using (var webResponse = await this.RestClientAsync.Get(uri))
                using (var stream = webResponse.GetResponseStream())
                    using (var reader = new StreamReader(stream))
                    {
                        var response = await reader.ReadToEndAsync();

                        return(new GetDatabaseResponse()
                        {
                            Database = this.JsonSerializer.Deserialize <DatabaseDetails>(response),
                            OpcRequestId = webResponse.Headers.Get("opc-request-id"),
                            ETag = webResponse.Headers.Get("ETag")
                        });
                    }
        }
Exemple #10
0
        public override async Task <GetDatabaseReply> GetDatabase(GetDatabaseRequest request, ServerCallContext callContext)
        {
            var tables = await _dbSchemaReader.GetDatabaseTablesAsync(new ConnectionInfo(request.ConnectionString), request.Database);

            var reply = new GetDatabaseReply();

            reply.Tables.AddRange(tables.Select(d =>
            {
                var table = new table {
                    Name = d.TableName, Schema = d.SchemaName
                };
                table.Fields.AddRange(d.Fields.Select(f => new field {
                    Name = f.Name, Type = f.Type
                }));

                return(table);
            }));

            return(reply);
        }
Exemple #11
0
        private void HandleOutput(GetDatabaseRequest request)
        {
            var waiterConfig = new WaiterConfiguration
            {
                MaxAttempts           = MaxWaitAttempts,
                GetNextDelayInSeconds = (_) => WaitIntervalSeconds
            };

            switch (ParameterSetName)
            {
            case LifecycleStateParamSet:
                response = client.Waiters.ForDatabase(request, waiterConfig, WaitForLifecycleState).Execute();
                break;

            case Default:
                response = client.GetDatabase(request).GetAwaiter().GetResult();
                break;
            }
            WriteOutput(response, response.Database);
        }
Exemple #12
0
        protected override void ProcessRecord()
        {
            base.ProcessRecord();
            GetDatabaseRequest request;

            try
            {
                request = new GetDatabaseRequest
                {
                    DatabaseId = DatabaseId
                };

                HandleOutput(request);
                FinishProcessing(response);
            }
            catch (Exception ex)
            {
                TerminatingErrorDuringExecution(ex);
            }
        }
Exemple #13
0
        public void GetDatabase2()
        {
            Mock <DatabaseAdmin.DatabaseAdminClient> mockGrpcClient = new Mock <DatabaseAdmin.DatabaseAdminClient>(MockBehavior.Strict);

            mockGrpcClient.Setup(x => x.CreateOperationsClient())
            .Returns(new Mock <Operations.OperationsClient>().Object);
            GetDatabaseRequest request = new GetDatabaseRequest
            {
                DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
            };
            Database expectedResponse = new Database
            {
                DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
            };

            mockGrpcClient.Setup(x => x.GetDatabase(request, It.IsAny <CallOptions>()))
            .Returns(expectedResponse);
            DatabaseAdminClient client   = new DatabaseAdminClientImpl(mockGrpcClient.Object, null);
            Database            response = client.GetDatabase(request);

            Assert.Same(expectedResponse, response);
            mockGrpcClient.VerifyAll();
        }
Exemple #14
0
        public async Task GetDatabaseAsync2()
        {
            Mock <DatabaseAdmin.DatabaseAdminClient> mockGrpcClient = new Mock <DatabaseAdmin.DatabaseAdminClient>(MockBehavior.Strict);

            mockGrpcClient.Setup(x => x.CreateOperationsClient())
            .Returns(new Mock <Operations.OperationsClient>().Object);
            GetDatabaseRequest request = new GetDatabaseRequest
            {
                DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
            };
            Database expectedResponse = new Database
            {
                DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
            };

            mockGrpcClient.Setup(x => x.GetDatabaseAsync(request, It.IsAny <CallOptions>()))
            .Returns(new Grpc.Core.AsyncUnaryCall <Database>(Task.FromResult(expectedResponse), null, null, null, null));
            DatabaseAdminClient client   = new DatabaseAdminClientImpl(mockGrpcClient.Object, null);
            Database            response = await client.GetDatabaseAsync(request);

            Assert.Same(expectedResponse, response);
            mockGrpcClient.VerifyAll();
        }
Exemple #15
0
 protected virtual string GenerateRelativeUrl(GetDatabaseRequest request)
 {
     return("/");
 }
Exemple #16
0
        /// <summary>
        /// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
        /// and applied to the table. This will end up creating a new table for each billing period.
        /// </summary>
        /// <param name="manifest"></param>
        /// <param name="context"></param>
        /// <returns>The table name</returns>
        private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
        {
            if (String.IsNullOrEmpty(_GlueDatabaseName))
            {
                string Message = "No Glue database name defined, cannot create a table.";
                context.LogWarning(Message);
                await SNSNotify(Message, context);

                return(String.Empty);
            }

            string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");

            string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);

            Dictionary <string, string> Parameters;
            StorageDescriptor           Descriptor;

            switch (Format)
            {
            case "csv":
            {
                Parameters = new Dictionary <string, string>()
                {
                    { "EXTERNAL", "TRUE" },
                    { "skip.header.line.count", "1" },
                    { "columnsOrdered", "true" },
                    { "compressionType", manifest.Compression.ToString().ToLower() },
                    { "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
                };

                Descriptor = new StorageDescriptor()
                {
                    Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
                        {
                            Name = $"{x.Category}/{x.Name}", Type = "string"
                        }).ToList(),
                    InputFormat  = "org.apache.hadoop.mapred.TextInputFormat",
                    OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
                    Location     = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
                    SerdeInfo    = new SerDeInfo()
                    {
                        Name = "OpenCSVSerde",
                        SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
                        Parameters           = new Dictionary <string, string>()
                        {
                            { "escapeChar", "\\" },
                            { "quoteChar", "\"" },
                            { "separatorChar", "," }
                        }
                    }
                };

                break;
            }

            case "parquet":
            {
                Parameters = new Dictionary <string, string>()
                {
                    { "EXTERNAL", "TRUE" },
                    { "compressionType", manifest.Compression.ToString().ToLower() },
                    { "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
                };

                Descriptor = new StorageDescriptor()
                {
                    Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
                        {
                            Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
                        }).ToList(),
                    InputFormat  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
                    OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
                    Location     = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
                    SerdeInfo    = new SerDeInfo()
                    {
                        Name = "ParquetHiveSerDe",
                        SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
                        Parameters           = new Dictionary <string, string>()
                        {
                            { "serialization.format", "1" }
                        }
                    }
                };
                break;
            }

            default:
            {
                string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
                await SNSNotify(Message, context);

                return(String.Empty);
            }
            }

            // The updated table input for this particular CUR
            TableInput TblInput = new TableInput()
            {
                Description       = Date,
                Name              = Date,
                TableType         = "EXTERNAL_TABLE",
                Parameters        = Parameters,
                StorageDescriptor = Descriptor
            };

            // Make sure the database exists
            GetDatabaseRequest GetDb = new GetDatabaseRequest()
            {
                Name = _GlueDatabaseName
            };

            try
            {
                await _GlueClient.GetDatabaseAsync(GetDb);

                context.LogInfo($"Database {_GlueDatabaseName} already exists.");
            }
            catch (EntityNotFoundException)
            {
                try
                {
                    CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
                    {
                        DatabaseInput = new DatabaseInput()
                        {
                            Name = _GlueDatabaseName
                        }
                    };

                    CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);

                    if (Response.HttpStatusCode == HttpStatusCode.OK)
                    {
                        context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
                    }
                    else
                    {
                        context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
                    }
                }
                catch (Exception ex)
                {
                    string Message = $"Failed to create the database {_GlueDatabaseName}.";
                    context.LogError(Message, ex);
                    await SNSNotify(Message + $" {ex.Message}", context);

                    return(String.Empty);
                }
            }

            // Make sure the table exists
            GetTableRequest GetTable = new GetTableRequest()
            {
                DatabaseName = _GlueDatabaseName,
                Name         = Date
            };

            try
            {
                GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);

                UpdateTableRequest UpdateReq = new UpdateTableRequest()
                {
                    TableInput   = TblInput,
                    DatabaseName = _GlueDatabaseName
                };

                UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);

                if (Response.HttpStatusCode == HttpStatusCode.OK)
                {
                    context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
                    return(TblInput.Name);
                }
                else
                {
                    string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
                    context.LogError(Message);
                    await SNSNotify(Message, context);

                    return(String.Empty);
                }
            }
            catch (EntityNotFoundException) // This means the table does not exist
            {
                CreateTableRequest CreateReq = new CreateTableRequest()
                {
                    TableInput   = TblInput,
                    DatabaseName = _GlueDatabaseName
                };

                CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);

                if (Response.HttpStatusCode == HttpStatusCode.OK)
                {
                    context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
                    return(TblInput.Name);
                }
                else
                {
                    string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
                    context.LogError(Message);
                    await SNSNotify(Message, context);

                    return(String.Empty);
                }
            }
        }
Exemple #17
0
 protected virtual HttpRequest CreateHttpRequest(GetDatabaseRequest request)
 {
     return(GetHttpRequestFactory.Create(request));
 }
Exemple #18
0
 protected override string GenerateRelativeUrl(GetDatabaseRequest request)
 {
     return(string.Format("/{0}", new UrlSegment(request.DbName)));
 }