public async Task CreateDatabase()
{
Console.WriteLine("Creating Database");
try
{
var createDatabaseRequest = new CreateDatabaseRequest
{
DatabaseName = Constants.DATABASE_NAME
};
CreateDatabaseResponse response = await writeClient.CreateDatabaseAsync(createDatabaseRequest);
Console.WriteLine($"Database {Constants.DATABASE_NAME} created");
}
catch (ConflictException)
{
Console.WriteLine("Database already exists.");
}
catch (Exception e)
{
Console.WriteLine("Create database failed:" + e.ToString());
}
}
public async Task CreateDatabase(IAmazonTimestreamWrite writeClient, string databaseName)
{
Console.WriteLine($"Creating Database {databaseName}");
try
{
var createDatabaseRequest = new CreateDatabaseRequest
{
DatabaseName = databaseName
};
await writeClient.CreateDatabaseAsync(createDatabaseRequest);
Console.WriteLine($"Database {databaseName} created");
}
catch (ConflictException)
{
Console.WriteLine("Database already exists.");
}
catch (Exception e)
{
Console.WriteLine($"Create database failed: {e}");
}
}
/// <summary>Snippet for CreateDatabaseAsync</summary>
public async Task CreateDatabaseAsync_RequestObject()
{
// Snippet: CreateDatabaseAsync(CreateDatabaseRequest,CallSettings)
// Create client
DatabaseAdminClient databaseAdminClient = await DatabaseAdminClient.CreateAsync();
// Initialize request argument(s)
CreateDatabaseRequest request = new CreateDatabaseRequest
{
ParentAsInstanceName = new InstanceName("[PROJECT]", "[INSTANCE]"),
CreateStatement = "",
};
// Make the request
Operation <Database, CreateDatabaseMetadata> response =
await databaseAdminClient.CreateDatabaseAsync(request);
// Poll until the returned long-running operation is complete
Operation <Database, CreateDatabaseMetadata> completedResponse =
await response.PollUntilCompletedAsync();
// Retrieve the operation result
Database result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Database, CreateDatabaseMetadata> retrievedResponse =
await databaseAdminClient.PollOnceCreateDatabaseAsync(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Database retrievedResult = retrievedResponse.Result;
}
// End snippet
}
/// <summary>
/// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
/// and applied to the table. This will end up creating a new table for each billing period.
/// </summary>
/// <param name="manifest"></param>
/// <param name="context"></param>
/// <returns>The table name</returns>
private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
{
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "No Glue database name defined, cannot create a table.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");
string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);
Dictionary <string, string> Parameters;
StorageDescriptor Descriptor;
switch (Format)
{
case "csv":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "skip.header.line.count", "1" },
{ "columnsOrdered", "true" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = $"{x.Category}/{x.Name}", Type = "string"
}).ToList(),
InputFormat = "org.apache.hadoop.mapred.TextInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "OpenCSVSerde",
SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
Parameters = new Dictionary <string, string>()
{
{ "escapeChar", "\\" },
{ "quoteChar", "\"" },
{ "separatorChar", "," }
}
}
};
break;
}
case "parquet":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
}).ToList(),
InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "ParquetHiveSerDe",
SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
Parameters = new Dictionary <string, string>()
{
{ "serialization.format", "1" }
}
}
};
break;
}
default:
{
string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
await SNSNotify(Message, context);
return(String.Empty);
}
}
// The updated table input for this particular CUR
TableInput TblInput = new TableInput()
{
Description = Date,
Name = Date,
TableType = "EXTERNAL_TABLE",
Parameters = Parameters,
StorageDescriptor = Descriptor
};
// Make sure the database exists
GetDatabaseRequest GetDb = new GetDatabaseRequest()
{
Name = _GlueDatabaseName
};
try
{
await _GlueClient.GetDatabaseAsync(GetDb);
context.LogInfo($"Database {_GlueDatabaseName} already exists.");
}
catch (EntityNotFoundException)
{
try
{
CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
{
DatabaseInput = new DatabaseInput()
{
Name = _GlueDatabaseName
}
};
CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
}
else
{
context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
}
}
catch (Exception ex)
{
string Message = $"Failed to create the database {_GlueDatabaseName}.";
context.LogError(Message, ex);
await SNSNotify(Message + $" {ex.Message}", context);
return(String.Empty);
}
}
// Make sure the table exists
GetTableRequest GetTable = new GetTableRequest()
{
DatabaseName = _GlueDatabaseName,
Name = Date
};
try
{
GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);
UpdateTableRequest UpdateReq = new UpdateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
catch (EntityNotFoundException) // This means the table does not exist
{
CreateTableRequest CreateReq = new CreateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
}
/// <summary>
/// 创建属于用户实例的数据库
/// </summary>
/// <param name="request">请求参数信息</param>
/// <returns>请求结果信息</returns>
public async Task <CreateDatabaseResponse> CreateDatabase(CreateDatabaseRequest request)
{
return(await new CreateDatabaseExecutor().Client(this).Execute <CreateDatabaseResponse, CreateDatabaseResult, CreateDatabaseRequest>(request).ConfigureAwait(false));
}
/// <summary>
/// 创建属于用户实例的数据库
/// </summary>
/// <param name="request">请求参数信息</param>
/// <returns>请求结果信息</returns>
public CreateDatabaseResponse CreateDatabase(CreateDatabaseRequest request)
{
return(new CreateDatabaseExecutor().Client(this).Execute <CreateDatabaseResponse, CreateDatabaseResult, CreateDatabaseRequest>(request));
}
