public void GetDatabaseResourceNames()
{
moq::Mock <FirestoreAdmin.FirestoreAdminClient> mockGrpcClient = new moq::Mock <FirestoreAdmin.FirestoreAdminClient>(moq::MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
GetDatabaseRequest request = new GetDatabaseRequest
{
DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
};
Database expectedResponse = new Database
{
DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
LocationId = "location_iddaa574e2",
Type = Database.Types.DatabaseType.Unspecified,
ConcurrencyMode = Database.Types.ConcurrencyMode.OptimisticWithEntityGroups,
Etag = "etage8ad7218",
};
mockGrpcClient.Setup(x => x.GetDatabase(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(expectedResponse);
FirestoreAdminClient client = new FirestoreAdminClientImpl(mockGrpcClient.Object, null);
Database response = client.GetDatabase(request.DatabaseName);
xunit::Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
public virtual async Task <GetDatabaseResponse> GetAsync(GetDatabaseRequest request)
{
var httpRequest = GetHttpRequestFactory.Create(request);
using (var httpResponse = await SendAsync(httpRequest).ForAwait())
return(await GetDatabaseResponseFactory.CreateAsync(httpResponse).ForAwait());
}
public async stt::Task GetDatabaseResourceNamesAsync()
{
moq::Mock <FirestoreAdmin.FirestoreAdminClient> mockGrpcClient = new moq::Mock <FirestoreAdmin.FirestoreAdminClient>(moq::MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
GetDatabaseRequest request = new GetDatabaseRequest
{
DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
};
Database expectedResponse = new Database
{
DatabaseName = DatabaseName.FromProjectDatabase("[PROJECT]", "[DATABASE]"),
LocationId = "location_iddaa574e2",
Type = Database.Types.DatabaseType.Unspecified,
ConcurrencyMode = Database.Types.ConcurrencyMode.OptimisticWithEntityGroups,
Etag = "etage8ad7218",
};
mockGrpcClient.Setup(x => x.GetDatabaseAsync(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(new grpccore::AsyncUnaryCall <Database>(stt::Task.FromResult(expectedResponse), null, null, null, null));
FirestoreAdminClient client = new FirestoreAdminClientImpl(mockGrpcClient.Object, null);
Database responseCallSettings = await client.GetDatabaseAsync(request.DatabaseName, gaxgrpc::CallSettings.FromCancellationToken(st::CancellationToken.None));
xunit::Assert.Same(expectedResponse, responseCallSettings);
Database responseCancellationToken = await client.GetDatabaseAsync(request.DatabaseName, st::CancellationToken.None);
xunit::Assert.Same(expectedResponse, responseCancellationToken);
mockGrpcClient.VerifyAll();
}
public virtual HttpRequest Create(GetDatabaseRequest request)
{
Ensure.Any.IsNotNull(request, nameof(request));
return(new HttpRequest(HttpMethod.Get, GenerateRelativeUrl(request))
.SetRequestTypeHeader(request.GetType()));
}
public virtual async Task <GetDatabaseResponse> GetAsync(CancellationToken cancellationToken = default)
{
var request = new GetDatabaseRequest(Connection.DbName);
var httpRequest = GetHttpRequestFactory.Create(request);
using (var httpResponse = await SendAsync(httpRequest, cancellationToken).ForAwait())
return(await GetDatabaseResponseFactory.CreateAsync(httpResponse).ForAwait());
}
public virtual async Task <GetDatabaseResponse> GetAsync(GetDatabaseRequest request)
{
var httpRequest = CreateHttpRequest(request);
using (var res = await SendAsync(httpRequest).ForAwait())
{
return(ProcessGetDatabaseResponse(res));
}
}
/// <summary>Snippet for GetDatabase</summary>
public void GetDatabase_RequestObject()
{
// Snippet: GetDatabase(GetDatabaseRequest,CallSettings)
// Create client
DatabaseAdminClient databaseAdminClient = DatabaseAdminClient.Create();
// Initialize request argument(s)
GetDatabaseRequest request = new GetDatabaseRequest
{
DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
};
// Make the request
Database response = databaseAdminClient.GetDatabase(request);
// End snippet
}
public async Task GetDatabaseAsync_RequestObject()
{
// Snippet: GetDatabaseAsync(GetDatabaseRequest,CallSettings)
// Create client
DatabaseAdminClient databaseAdminClient = await DatabaseAdminClient.CreateAsync();
// Initialize request argument(s)
GetDatabaseRequest request = new GetDatabaseRequest
{
Name = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]").ToString(),
};
// Make the request
Database response = await databaseAdminClient.GetDatabaseAsync(request);
// End snippet
}
/// <summary>
/// Gets information about a specific database.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <GetDatabaseResponse> GetDatabase(GetDatabaseRequest request)
{
var uri = new Uri($"{GetEndPoint(DatabaseServices.Databases, this.Region)}/{request.DatabaseId}");
using (var webResponse = await this.RestClientAsync.Get(uri))
using (var stream = webResponse.GetResponseStream())
using (var reader = new StreamReader(stream))
{
var response = await reader.ReadToEndAsync();
return(new GetDatabaseResponse()
{
Database = this.JsonSerializer.Deserialize <DatabaseDetails>(response),
OpcRequestId = webResponse.Headers.Get("opc-request-id"),
ETag = webResponse.Headers.Get("ETag")
});
}
}
public override async Task <GetDatabaseReply> GetDatabase(GetDatabaseRequest request, ServerCallContext callContext)
{
var tables = await _dbSchemaReader.GetDatabaseTablesAsync(new ConnectionInfo(request.ConnectionString), request.Database);
var reply = new GetDatabaseReply();
reply.Tables.AddRange(tables.Select(d =>
{
var table = new table {
Name = d.TableName, Schema = d.SchemaName
};
table.Fields.AddRange(d.Fields.Select(f => new field {
Name = f.Name, Type = f.Type
}));
return(table);
}));
return(reply);
}
private void HandleOutput(GetDatabaseRequest request)
{
var waiterConfig = new WaiterConfiguration
{
MaxAttempts = MaxWaitAttempts,
GetNextDelayInSeconds = (_) => WaitIntervalSeconds
};
switch (ParameterSetName)
{
case LifecycleStateParamSet:
response = client.Waiters.ForDatabase(request, waiterConfig, WaitForLifecycleState).Execute();
break;
case Default:
response = client.GetDatabase(request).GetAwaiter().GetResult();
break;
}
WriteOutput(response, response.Database);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
GetDatabaseRequest request;
try
{
request = new GetDatabaseRequest
{
DatabaseId = DatabaseId
};
HandleOutput(request);
FinishProcessing(response);
}
catch (Exception ex)
{
TerminatingErrorDuringExecution(ex);
}
}
public void GetDatabase2()
{
Mock <DatabaseAdmin.DatabaseAdminClient> mockGrpcClient = new Mock <DatabaseAdmin.DatabaseAdminClient>(MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient())
.Returns(new Mock <Operations.OperationsClient>().Object);
GetDatabaseRequest request = new GetDatabaseRequest
{
DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
};
Database expectedResponse = new Database
{
DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
};
mockGrpcClient.Setup(x => x.GetDatabase(request, It.IsAny <CallOptions>()))
.Returns(expectedResponse);
DatabaseAdminClient client = new DatabaseAdminClientImpl(mockGrpcClient.Object, null);
Database response = client.GetDatabase(request);
Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
public async Task GetDatabaseAsync2()
{
Mock <DatabaseAdmin.DatabaseAdminClient> mockGrpcClient = new Mock <DatabaseAdmin.DatabaseAdminClient>(MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient())
.Returns(new Mock <Operations.OperationsClient>().Object);
GetDatabaseRequest request = new GetDatabaseRequest
{
DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
};
Database expectedResponse = new Database
{
DatabaseName = new DatabaseName("[PROJECT]", "[INSTANCE]", "[DATABASE]"),
};
mockGrpcClient.Setup(x => x.GetDatabaseAsync(request, It.IsAny <CallOptions>()))
.Returns(new Grpc.Core.AsyncUnaryCall <Database>(Task.FromResult(expectedResponse), null, null, null, null));
DatabaseAdminClient client = new DatabaseAdminClientImpl(mockGrpcClient.Object, null);
Database response = await client.GetDatabaseAsync(request);
Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
protected virtual string GenerateRelativeUrl(GetDatabaseRequest request)
{
return("/");
}
/// <summary>
/// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
/// and applied to the table. This will end up creating a new table for each billing period.
/// </summary>
/// <param name="manifest"></param>
/// <param name="context"></param>
/// <returns>The table name</returns>
private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
{
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "No Glue database name defined, cannot create a table.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");
string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);
Dictionary <string, string> Parameters;
StorageDescriptor Descriptor;
switch (Format)
{
case "csv":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "skip.header.line.count", "1" },
{ "columnsOrdered", "true" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = $"{x.Category}/{x.Name}", Type = "string"
}).ToList(),
InputFormat = "org.apache.hadoop.mapred.TextInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "OpenCSVSerde",
SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
Parameters = new Dictionary <string, string>()
{
{ "escapeChar", "\\" },
{ "quoteChar", "\"" },
{ "separatorChar", "," }
}
}
};
break;
}
case "parquet":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
}).ToList(),
InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "ParquetHiveSerDe",
SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
Parameters = new Dictionary <string, string>()
{
{ "serialization.format", "1" }
}
}
};
break;
}
default:
{
string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
await SNSNotify(Message, context);
return(String.Empty);
}
}
// The updated table input for this particular CUR
TableInput TblInput = new TableInput()
{
Description = Date,
Name = Date,
TableType = "EXTERNAL_TABLE",
Parameters = Parameters,
StorageDescriptor = Descriptor
};
// Make sure the database exists
GetDatabaseRequest GetDb = new GetDatabaseRequest()
{
Name = _GlueDatabaseName
};
try
{
await _GlueClient.GetDatabaseAsync(GetDb);
context.LogInfo($"Database {_GlueDatabaseName} already exists.");
}
catch (EntityNotFoundException)
{
try
{
CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
{
DatabaseInput = new DatabaseInput()
{
Name = _GlueDatabaseName
}
};
CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
}
else
{
context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
}
}
catch (Exception ex)
{
string Message = $"Failed to create the database {_GlueDatabaseName}.";
context.LogError(Message, ex);
await SNSNotify(Message + $" {ex.Message}", context);
return(String.Empty);
}
}
// Make sure the table exists
GetTableRequest GetTable = new GetTableRequest()
{
DatabaseName = _GlueDatabaseName,
Name = Date
};
try
{
GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);
UpdateTableRequest UpdateReq = new UpdateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
catch (EntityNotFoundException) // This means the table does not exist
{
CreateTableRequest CreateReq = new CreateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
}
protected virtual HttpRequest CreateHttpRequest(GetDatabaseRequest request)
{
return(GetHttpRequestFactory.Create(request));
}
protected override string GenerateRelativeUrl(GetDatabaseRequest request)
{
return(string.Format("/{0}", new UrlSegment(request.DbName)));
}
