public static async void UpdateTableDemo(string tableName)
{
try
{
using (IAmazonDynamoDB client = GetDynamoDbClient())
{
UpdateTableRequest updateTableRequest = new UpdateTableRequest();
updateTableRequest.TableName = tableName;
updateTableRequest.ProvisionedThroughput = new ProvisionedThroughput()
{
ReadCapacityUnits = 2, WriteCapacityUnits = 2
};
UpdateTableResponse updateTableResponse = await client.UpdateTableAsync(updateTableRequest);
TableDescription tableDescription = updateTableResponse.TableDescription;
Debug.WriteLine(string.Format("Update table command sent to Amazon for table {0}, status after update: {1}", tableName
, tableDescription.TableStatus));
}
}
catch (AmazonDynamoDBException exception)
{
Debug.WriteLine(string.Concat("Exception while updating DynamoDb table: {0}", exception.Message));
Debug.WriteLine(String.Concat("Error code: {0}, error type: {1}", exception.ErrorCode, exception.ErrorType));
}
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
UpdateTableResponse response = new UpdateTableResponse();
return(response);
}
public async Task UpdateTable()
{
Console.WriteLine("Updating Table");
try
{
var updateTableRequest = new UpdateTableRequest
{
DatabaseName = Constants.DATABASE_NAME,
TableName = Constants.TABLE_NAME,
RetentionProperties = new RetentionProperties
{
MagneticStoreRetentionPeriodInDays = Constants.CT_TTL_DAYS,
MemoryStoreRetentionPeriodInHours = Constants.HT_TTL_HOURS
}
};
UpdateTableResponse response = await writeClient.UpdateTableAsync(updateTableRequest);
Console.WriteLine($"Table {Constants.TABLE_NAME} updated");
}
catch (ResourceNotFoundException)
{
Console.WriteLine("Table does not exist.");
}
catch (Exception e)
{
Console.WriteLine("Update table failed:" + e.ToString());
}
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
UpdateTableResponse response = new UpdateTableResponse();
context.Read();
response.UpdateTableResult = UpdateTableResultUnmarshaller.GetInstance().Unmarshall(context);
return(response);
}
public static void TableOperations()
{
// 创建表
OTSClient otsClient = Config.GetClient();
{
Console.WriteLine("Start create table...");
PrimaryKeySchema primaryKeySchema = new PrimaryKeySchema
{
{ "pk0", ColumnValueType.Integer },
{ "pk1", ColumnValueType.String }
};
TableMeta tableMeta = new TableMeta(TableName, primaryKeySchema);
CapacityUnit reservedThroughput = new CapacityUnit(0, 0);
CreateTableRequest request = new CreateTableRequest(tableMeta, reservedThroughput);
otsClient.CreateTable(request);
Console.WriteLine("Table is created: " + TableName);
}
// 更新表
{
Thread.Sleep(60 * 1000); // 每次更新表需要至少间隔1分钟
Console.WriteLine("Start update table...");
CapacityUnit reservedThroughput = new CapacityUnit(0, 0); // 将预留CU调整为0，0
UpdateTableRequest request = new UpdateTableRequest(TableName, reservedThroughput);
UpdateTableResponse response = otsClient.UpdateTable(request);
Console.WriteLine("LastIncreaseTime: " + response.ReservedThroughputDetails.LastIncreaseTime);
Console.WriteLine("LastDecreaseTime: " + response.ReservedThroughputDetails.LastDecreaseTime);
Console.WriteLine("NumberOfDecreaseToday: " + response.ReservedThroughputDetails.LastIncreaseTime);
Console.WriteLine("ReadCapacity: " + response.ReservedThroughputDetails.CapacityUnit.Read);
Console.WriteLine("WriteCapacity: " + response.ReservedThroughputDetails.CapacityUnit.Write);
}
// 描述表
{
Console.WriteLine("Start describe table...");
DescribeTableRequest request = new DescribeTableRequest(TableName);
DescribeTableResponse response = otsClient.DescribeTable(request);
Console.WriteLine("LastIncreaseTime: " + response.ReservedThroughputDetails.LastIncreaseTime);
Console.WriteLine("LastDecreaseTime: " + response.ReservedThroughputDetails.LastDecreaseTime);
Console.WriteLine("NumberOfDecreaseToday: " + response.ReservedThroughputDetails.LastIncreaseTime);
Console.WriteLine("ReadCapacity: " + response.ReservedThroughputDetails.CapacityUnit.Read);
Console.WriteLine("WriteCapacity: " + response.ReservedThroughputDetails.CapacityUnit.Write);
}
// 删除表
{
Console.WriteLine("Start delete table...");
DeleteTableRequest request = new DeleteTableRequest(TableName);
otsClient.DeleteTable(request);
Console.WriteLine("Table is deleted.");
}
}
public static UpdateTableResponse Unmarshall(UnmarshallerContext _ctx)
{
UpdateTableResponse updateTableResponse = new UpdateTableResponse();
updateTableResponse.HttpResponse = _ctx.HttpResponse;
updateTableResponse.RequestId = _ctx.StringValue("UpdateTable.RequestId");
UpdateTableResponse.UpdateTable_TaskInfo taskInfo = new UpdateTableResponse.UpdateTable_TaskInfo();
taskInfo.TaskId = _ctx.StringValue("UpdateTable.TaskInfo.TaskId");
taskInfo.Content = _ctx.StringValue("UpdateTable.TaskInfo.Content");
taskInfo.Status = _ctx.StringValue("UpdateTable.TaskInfo.Status");
taskInfo.NextTaskId = _ctx.StringValue("UpdateTable.TaskInfo.NextTaskId");
updateTableResponse.TaskInfo = taskInfo;
return(updateTableResponse);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
UpdateTableResponse response = new UpdateTableResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("TableDescription", targetDepth))
{
response.TableDescription = TableDescriptionUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
}
return(response);
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
UpdateTableResponse response = new UpdateTableResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("resourceArn", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.ResourceArn = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
private static void UnmarshallResult(JsonUnmarshallerContext context, UpdateTableResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
while (context.Read())
{
if (context.TestExpression("TableDescription", targetDepth))
{
context.Read();
response.TableDescription = TableDescriptionUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.CurrentDepth <= originalDepth)
{
return;
}
}
return;
}
public async Task FunctionHandler(S3Event s3event, ILambdaContext context)
{
if (s3event != null)
{
foreach (var record in s3event.Records)
{
context.Logger.LogLine(record.S3.Bucket.Name);
string bucket = record.S3.Bucket.Name.Substring(14).Replace('-', ' ');
context.Logger.LogLine(bucket);
if (LogContextOperations.IfInputExist(bucket))
{
LogInput retrievedLI = LogContextOperations.GetLogInputByName(bucket);
GlueConsolidatedEntity retrievedGCE = LogContextOperations.GetGlueConsolidatedEntity(retrievedLI.ID);
GlueDatabaseTable retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
context.Logger.LogLine(retrievedLI.ID + " | " + retrievedLI.Name);
if (retrievedLI.InitialIngest == false && retrievedGCE == null)
{
context.Logger.LogLine("Log Input has not be crawled before and has no crawler");
CreateCrawlerResponse createCrawlerResponse = await GlueClient.CreateCrawlerAsync(new CreateCrawlerRequest
{
Name = retrievedLI.Name,
DatabaseName = LogContextOperations.GetGlueDatabase().Name,
Role = "GlueServiceRole",
SchemaChangePolicy = new SchemaChangePolicy
{
DeleteBehavior = DeleteBehavior.DEPRECATE_IN_DATABASE,
UpdateBehavior = UpdateBehavior.UPDATE_IN_DATABASE
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
Targets = new CrawlerTargets
{
S3Targets = new List <S3Target>
{
new S3Target
{
Path = "s3://" + LogContextOperations.GetInputS3BucketName(retrievedLI.ID)
}
}
}
});
if (createCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Created");
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedLI.Name
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Just Created Started");
LogContextOperations.AddGlueConsolidatedEntity(new GlueConsolidatedEntity
{
CrawlerName = retrievedLI.Name,
LinkedLogInputID = retrievedLI.ID
});
}
}
}
else if (retrievedLI.InitialIngest == false && retrievedGCE != null)
{
context.Logger.LogLine("Log Input has not be crawled before but has a crawler");
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY) && getCrawlerResponse.Crawler.LastCrawl == null)
{
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Started");
}
}
else if (getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY) && getCrawlerResponse.Crawler.LastCrawl != null)
{
context.Logger.LogLine("Log Input has been crawled before, has a crawler but not a job");
LogContextOperations.AddGlueDatabaseTable(new GlueDatabaseTable
{
LinkedDatabaseID = 1,
LinkedGlueConsolidatedInputEntityID = retrievedGCE.ID,
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
CreateJobRequest createJobRequest = new CreateJobRequest
{
Name = retrievedLI.Name,
DefaultArguments = new Dictionary <string, string>
{
{ "--enable-spark-ui", "true" },
{ "--spark-event-logs-path", "s3://aws-glue-spark-188363912800-ap-southeast-1" },
{ "--job-bookmark-option", "job-bookmark-enable" },
{ "--job-language", "python" },
{ "--TempDir", "s3://aws-glue-temporary-188363912800-ap-southeast-1/root" },
{ "--TABLE_NAME", retrievedGDT.Name }
},
MaxCapacity = 10.0,
Role = "GlueServiceRole",
Connections = new ConnectionsList
{
Connections = new List <string>
{
"SmartInsights"
}
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
MaxRetries = 0,
GlueVersion = "1.0",
ExecutionProperty = new ExecutionProperty
{
MaxConcurrentRuns = 1
},
Timeout = 2880
};
if (retrievedLI.LogInputCategory.Equals(LogInputCategory.ApacheWebServer))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Apache CLF"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SquidProxy))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Cisco Squid Proxy"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SSH))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Splunk SSH"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.WindowsEventLogs))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Windows Events"
};
}
CreateJobResponse createJobResponse = await GlueClient.CreateJobAsync(createJobRequest);
if (createJobResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Created");
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = createJobResponse.Name,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Just Created Started");
retrievedGCE.JobName = createJobResponse.Name;
LogContextOperations.UpdateGlueConsolidatedEntity(retrievedGCE);
LogContextOperations.UpdateInputIngestionStatus(bucket);
}
}
}
}
}
}
else if (retrievedLI.InitialIngest == true)
{
context.Logger.LogLine("Log Input has been crawled before");
context.Logger.LogLine(retrievedGCE.JobName);
if (retrievedGCE.JobName == null && retrievedGDT == null)
{
context.Logger.LogLine("Log Input has not be transferred over to RDS before due to no job");
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
LogContextOperations.AddGlueDatabaseTable(new GlueDatabaseTable
{
LinkedDatabaseID = 1,
LinkedGlueConsolidatedInputEntityID = retrievedGCE.ID,
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
CreateJobRequest createJobRequest = new CreateJobRequest
{
Name = retrievedLI.Name,
DefaultArguments = new Dictionary <string, string>
{
{ "--enable-spark-ui", "true" },
{ "--spark-event-logs-path", "s3://aws-glue-spark-188363912800-ap-southeast-1" },
{ "--job-bookmark-option", "job-bookmark-enable" },
{ "--job-language", "python" },
{ "--TempDir", "s3://aws-glue-temporary-188363912800-ap-southeast-1/root" },
{ "--TABLE_NAME", retrievedGDT.Name }
},
MaxCapacity = 10.0,
Role = "GlueServiceRole",
Connections = new ConnectionsList
{
Connections = new List <string>
{
"SmartInsights"
}
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
MaxRetries = 0,
GlueVersion = "1.0",
ExecutionProperty = new ExecutionProperty
{
MaxConcurrentRuns = 1
},
Timeout = 2880
};
if (retrievedLI.LogInputCategory.Equals(LogInputCategory.ApacheWebServer))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Apache CLF"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SquidProxy))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Cisco Squid Proxy"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SSH))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Splunk SSH"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.WindowsEventLogs))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Windows Events"
};
}
CreateJobResponse createJobResponse = await GlueClient.CreateJobAsync(createJobRequest);
if (createJobResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Created");
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = createJobResponse.Name,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Just Created Started");
retrievedGCE.JobName = createJobResponse.Name;
LogContextOperations.UpdateGlueConsolidatedEntity(retrievedGCE);
LogContextOperations.UpdateInputIngestionStatus(bucket);
}
}
}
}
}
}
else
{
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK) && getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY))
{
if ((getCrawlerResponse.Crawler.LastCrawl.StartTime.Hour < DateTime.Now.Hour && getCrawlerResponse.Crawler.LastCrawl.StartTime.Day == DateTime.Now.Day) || getCrawlerResponse.Crawler.LastCrawl.StartTime.Day != DateTime.Now.Day)
{
context.Logger.LogLine("Log Input has been transferred over to RDS before but time condition not met");
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Started");
}
}
else
{
context.Logger.LogLine("Log Input has been transferred over to RDS before and time condition met");
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Table updated before running job");
context.Logger.LogLine(retrievedGCE.JobName);
GetJobRunsResponse getJobRunsResponse = await GlueClient.GetJobRunsAsync(new GetJobRunsRequest
{
JobName = retrievedGCE.JobName
});
bool jobRunning = false;
context.Logger.LogLine(getJobRunsResponse.JobRuns.Count().ToString());
foreach (JobRun j in getJobRunsResponse.JobRuns)
{
context.Logger.LogLine(j.Id + " | " + j.JobRunState);
if (j.JobRunState.Equals(JobRunState.STARTING) || j.JobRunState.Equals(JobRunState.RUNNING) || j.JobRunState.Equals(JobRunState.STOPPING))
{
jobRunning = true;
break;
}
}
context.Logger.LogLine(jobRunning.ToString());
if (!jobRunning)
{
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = retrievedGCE.JobName,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Started");
}
}
}
}
}
}
}
}
}
}
}
}
/// <summary>
/// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
/// and applied to the table. This will end up creating a new table for each billing period.
/// </summary>
/// <param name="manifest"></param>
/// <param name="context"></param>
/// <returns>The table name</returns>
private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
{
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "No Glue database name defined, cannot create a table.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");
string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);
Dictionary <string, string> Parameters;
StorageDescriptor Descriptor;
switch (Format)
{
case "csv":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "skip.header.line.count", "1" },
{ "columnsOrdered", "true" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = $"{x.Category}/{x.Name}", Type = "string"
}).ToList(),
InputFormat = "org.apache.hadoop.mapred.TextInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "OpenCSVSerde",
SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
Parameters = new Dictionary <string, string>()
{
{ "escapeChar", "\\" },
{ "quoteChar", "\"" },
{ "separatorChar", "," }
}
}
};
break;
}
case "parquet":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
}).ToList(),
InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "ParquetHiveSerDe",
SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
Parameters = new Dictionary <string, string>()
{
{ "serialization.format", "1" }
}
}
};
break;
}
default:
{
string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
await SNSNotify(Message, context);
return(String.Empty);
}
}
// The updated table input for this particular CUR
TableInput TblInput = new TableInput()
{
Description = Date,
Name = Date,
TableType = "EXTERNAL_TABLE",
Parameters = Parameters,
StorageDescriptor = Descriptor
};
// Make sure the database exists
GetDatabaseRequest GetDb = new GetDatabaseRequest()
{
Name = _GlueDatabaseName
};
try
{
await _GlueClient.GetDatabaseAsync(GetDb);
context.LogInfo($"Database {_GlueDatabaseName} already exists.");
}
catch (EntityNotFoundException)
{
try
{
CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
{
DatabaseInput = new DatabaseInput()
{
Name = _GlueDatabaseName
}
};
CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
}
else
{
context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
}
}
catch (Exception ex)
{
string Message = $"Failed to create the database {_GlueDatabaseName}.";
context.LogError(Message, ex);
await SNSNotify(Message + $" {ex.Message}", context);
return(String.Empty);
}
}
// Make sure the table exists
GetTableRequest GetTable = new GetTableRequest()
{
DatabaseName = _GlueDatabaseName,
Name = Date
};
try
{
GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);
UpdateTableRequest UpdateReq = new UpdateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
catch (EntityNotFoundException) // This means the table does not exist
{
CreateTableRequest CreateReq = new CreateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
}
