public async Task <IEnumerable <S3Object> > ListObjects(S3ListParameter param)
{
var startTimeUtc = param.StartTime.ToUniversalTime();
var endTimeUtc = param.EndTime.ToUniversalTime();
var hashToken = await GetHashToken(param);
var items = new List <S3Object>();
var startTimeKey = $"{param.RootDirectory}/{hashToken}.{startTimeUtc:yyyy-MM-dd-HH}.00000000";
var endTimeKey = $"{param.RootDirectory}/{hashToken}.{endTimeUtc:yyyy-MM-dd-HH}.99999999";
var prefixDate = startTimeUtc.Date;
while (prefixDate < endTimeUtc)
{
// Query condition is only prefix, so retrive one day log files at a time and filter.
var subItems = await ListObjects(param.BucketName,
$"{param.RootDirectory}/{hashToken}.{prefixDate:yyyy-MM-dd}");
items.AddRange(subItems.Where(i =>
i.Key.CompareTo(startTimeKey) >= 0 && i.Key.CompareTo(endTimeKey) <= 0));
prefixDate = prefixDate.AddDays(1);
}
return(items);
}
/// <summary>
/// Get {HASH_TOKEN} string.
/// https://s3-ap-northeast-1.amazonaws.com/{BUCKET_NAME}/{ROOT_DIRECTORY}/{HASH_TOKEN}.YYYY-mm-DD-HH.9281d331.gz
///
/// HASH_TOKEN is determinted by the value of BUCKET_NAME and ROOT_DIRECTORY.
/// </summary>
/// <param name="param"></param>
/// <returns></returns>
public async Task <string> GetHashToken(S3ListParameter param)
{
using (var client = CreateClient())
{
var prefix = param.RootDirectory + "/";
var request = new ListObjectsV2Request
{
BucketName = param.BucketName,
Prefix = prefix,
MaxKeys = 1
};
var response = await client.ListObjectsV2Async(request);
var key = response?.S3Objects.FirstOrDefault()?.Key;
if (key == null)
{
return(null);
}
key = key.Substring(prefix.Length);
return(key.Substring(0, key.IndexOf('.')));
}
}
public override void Execute()
{
var sw = Stopwatch.StartNew();
var connectionString = ConfigurationManager.ConnectionStrings["default"]?.ConnectionString;
if (string.IsNullOrEmpty(connectionString))
{
throw new ApplicationException("ConnectionString must be required in configuration file.");
}
_database = new DatabaseManager(connectionString);
_s3 = new S3Accessor("cflogget");
_logParser = new CFLogParser();
var param = new S3ListParameter
{
BucketName = BucketName,
RootDirectory = RootDirectory,
StartTime = DateTime.Parse(StartDateTime),
EndTime = DateTime.Parse(EndDateTime)
};
var items = _s3.ListObjects(param).Result;
var continueDownload = Prompt.GetYesNo($"Target S3 Objects is {items.Count()} count, {items.Sum(i => i.Size):#,#} byte. Continue downloading log files ?", true);
if (!continueDownload)
{
return;
}
_tableName = $"{RootDirectory.Replace('-', '_').Replace('/', '_')}_{DateTime.Now:MMdd_HHmm}";
_database.CreateTable(_tableName);
//ExecuteWhenAll(items);
ExecuteParallel(items);
var count = _database.Count(_tableName);
Console.WriteLine($"\n*** Completed to get logs.\n Log record count is {count}.\n Logs was stored in '{_tableName}' table.\n Elapsed: {sw.Elapsed}");
}