Esempio n. 1
0
        public async Task <IEnumerable <S3Object> > ListObjects(S3ListParameter param)
        {
            var startTimeUtc = param.StartTime.ToUniversalTime();
            var endTimeUtc   = param.EndTime.ToUniversalTime();
            var hashToken    = await GetHashToken(param);

            var items = new List <S3Object>();

            var startTimeKey = $"{param.RootDirectory}/{hashToken}.{startTimeUtc:yyyy-MM-dd-HH}.00000000";
            var endTimeKey   = $"{param.RootDirectory}/{hashToken}.{endTimeUtc:yyyy-MM-dd-HH}.99999999";
            var prefixDate   = startTimeUtc.Date;

            while (prefixDate < endTimeUtc)
            {
                // Query condition is only prefix, so retrive one day log files at a time and filter.
                var subItems = await ListObjects(param.BucketName,
                                                 $"{param.RootDirectory}/{hashToken}.{prefixDate:yyyy-MM-dd}");

                items.AddRange(subItems.Where(i =>
                                              i.Key.CompareTo(startTimeKey) >= 0 && i.Key.CompareTo(endTimeKey) <= 0));

                prefixDate = prefixDate.AddDays(1);
            }

            return(items);
        }
Esempio n. 2
0
        /// <summary>
        /// Get {HASH_TOKEN} string.
        /// https://s3-ap-northeast-1.amazonaws.com/{BUCKET_NAME}/{ROOT_DIRECTORY}/{HASH_TOKEN}.YYYY-mm-DD-HH.9281d331.gz
        ///
        /// HASH_TOKEN is determinted by the value of BUCKET_NAME and ROOT_DIRECTORY.
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        public async Task <string> GetHashToken(S3ListParameter param)
        {
            using (var client = CreateClient())
            {
                var prefix = param.RootDirectory + "/";

                var request = new ListObjectsV2Request
                {
                    BucketName = param.BucketName,
                    Prefix     = prefix,
                    MaxKeys    = 1
                };

                var response = await client.ListObjectsV2Async(request);

                var key = response?.S3Objects.FirstOrDefault()?.Key;
                if (key == null)
                {
                    return(null);
                }

                key = key.Substring(prefix.Length);
                return(key.Substring(0, key.IndexOf('.')));
            }
        }
Esempio n. 3
0
        public override void Execute()
        {
            var sw = Stopwatch.StartNew();
            var connectionString = ConfigurationManager.ConnectionStrings["default"]?.ConnectionString;

            if (string.IsNullOrEmpty(connectionString))
            {
                throw new ApplicationException("ConnectionString must be required in configuration file.");
            }

            _database  = new DatabaseManager(connectionString);
            _s3        = new S3Accessor("cflogget");
            _logParser = new CFLogParser();

            var param = new S3ListParameter
            {
                BucketName    = BucketName,
                RootDirectory = RootDirectory,
                StartTime     = DateTime.Parse(StartDateTime),
                EndTime       = DateTime.Parse(EndDateTime)
            };
            var items = _s3.ListObjects(param).Result;

            var continueDownload = Prompt.GetYesNo($"Target S3 Objects is {items.Count()} count, {items.Sum(i => i.Size):#,#} byte. Continue downloading log files ?", true);

            if (!continueDownload)
            {
                return;
            }

            _tableName = $"{RootDirectory.Replace('-', '_').Replace('/', '_')}_{DateTime.Now:MMdd_HHmm}";
            _database.CreateTable(_tableName);

            //ExecuteWhenAll(items);
            ExecuteParallel(items);

            var count = _database.Count(_tableName);

            Console.WriteLine($"\n*** Completed to get logs.\n Log record count is {count}.\n Logs was stored in '{_tableName}' table.\n Elapsed: {sw.Elapsed}");
        }