Represents an S3 Object. Contains all attributes that an S3 Object has. For more information about S3 Objects refer:
Esempio n. 1
0
        private void retrieveFile(S3Object o, string filename)
        {
            DateTime lastModified = o.LastModified;
            FileInfo fi = null;

            if (File.Exists(filename))
            {
                fi = new FileInfo(filename);
                if (fi.LastWriteTime < lastModified)
                {
                    if ( !OnlyCount ) transferUtility.Download(filename, Bucket, o.Key);
                    Console.WriteLine("Update.. {0}", filename);
                    CopiedFiles.Add(filename);
                }
                else
                {
                   // Console.WriteLine("Skip.... {0}", filename);
                }
            }
            else
            {
                if (!OnlyCount)
                {
                    transferUtility.Download(filename, Bucket, o.Key);
                }
                Console.WriteLine("Create.. {0}", filename);
                CopiedFiles.Add(filename);
            }

            /// カウントモードでなければ、タイムスタンプを再設定する
            fi = new FileInfo(filename);
            if ( lastModified != DateTime.MinValue && !OnlyCount )
                fi.LastWriteTime = lastModified;
        }
Esempio n. 2
0
        private void ProcessFile(S3Object file)
        {
            Trace.WriteLine(string.Format("Processing item {0}", file.Key));

            var lines = ReadS3File(file);

            var trimmedDate = file.Key.Substring(DATE_EXTRACTION_PREFIX.Length, 11);
            var date = DateTime.ParseExact(trimmedDate, FORMAT, PROVIDER);

            var fileMod = file.GetHashCode() % ComputeNode.GlobalBucketCount;
            var buckets = ComputeNode.Catalogs.Values.Cast<ICatalog>().Where(c => c.CatalogName == CATALOG).First().Buckets;
            var bucketMod = buckets.First(b => b.Value.BucketMod == fileMod).Value;

            Trace.WriteLine(string.Format("Adding data items from {0}", file.Key));
            lines.AsParallel().ForAll(line =>
                {
                    var items = line.Split(' ');
                    Debug.Assert(items.Length == 4);

                    var projectCode = HttpUtility.UrlDecode(items[0]);
                    var pageName = HttpUtility.UrlDecode(items[1]);
                    var pageViews = int.Parse(items[2]);
                    var pageSizeKB = long.Parse(items[3]);

                    var wikiStat = new WikipediaHourlyPageStats(date, projectCode, pageName, pageViews, pageSizeKB);
                    bucketMod.BucketDataTables[TABLE].AddItem(wikiStat);
                });

            Trace.WriteLine(string.Format("Added data items from {0}", file.Key));
        }
 /// <summary>
 /// Sets the storage class for the S3 Object to the value
 /// specified.
 /// </summary>
 /// <param name="s3Object">The S3 Object whose storage class needs changing</param>
 /// <param name="sClass">The new Storage Class for the object</param>
 /// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
 /// <seealso cref="T:Amazon.S3.Model.S3StorageClass"/>
 public static void SetObjectStorageClass(S3Object s3Object, S3StorageClass sClass, AmazonS3 s3Client)
 {
     SetObjectStorageClass(s3Object.BucketName, s3Object.Key, sClass, s3Client);
 }
        private TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S3Object s3Object, int prefixLength)
        {
            var downloadRequest = new TransferUtilityDownloadRequest();
            downloadRequest.BucketName = this._request.BucketName;
            downloadRequest.Key = s3Object.Key;
            var file = s3Object.Key.Substring(prefixLength).Replace('/','\\');
            downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, file);
            downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;

            return downloadRequest;
        }
 public S3StorageFolder(S3Object entry, long folderSize)
 {
     // TODO: Complete member initialization
     this._folderSize = folderSize;
     this._entry = entry;
 }
Esempio n. 6
0
 /// <summary>
 /// Initializes a new instance of the FileObject class.
 /// </summary>
 /// <param name="s3Object">The Amazon S3 object to create this instance from.</param>
 public FileObject(S3Object s3Object)
 {
     this.Key = s3Object.Key;
     this.LastModified = DateTime.Parse(s3Object.LastModified, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime();
     this.LastModifiedGMTString = String.Format(CultureInfo.InvariantCulture, "{0:s}Z", this.LastModified);
     this.Size = Int64.Parse(s3Object.Size, CultureInfo.InvariantCulture);
 }
Esempio n. 7
0
 /// <summary>
 /// Sets the redirect location for the S3 Object's when being accessed through the S3 website endpoint.
 /// </summary>
 /// <param name="s3Object">The S3 Object</param>
 /// <param name="websiteRedirectLocation">The redirect location</param>
 /// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
 public static void SetWebsiteRedirectLocation(S3Object s3Object, string websiteRedirectLocation, AmazonS3 s3Client)
 {
     SetWebsiteRedirectLocation(s3Object.BucketName, s3Object.Key, websiteRedirectLocation, s3Client);
 }
Esempio n. 8
0
 /// <summary>
 /// Sets the server side encryption method for the S3 Object to the value
 /// specified.
 /// </summary>
 /// <param name="s3Object">The S3 Object</param>
 /// <param name="method">The server side encryption method</param>
 /// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
 public static void SetServerSideEncryption(S3Object s3Object, ServerSideEncryptionMethod method, AmazonS3 s3Client)
 {
     SetServerSideEncryption(s3Object.BucketName, s3Object.Key, method, s3Client);
 }
Esempio n. 9
0
 public S3StorageFile(S3Object entry, string contentType)
 {
     // TODO: Complete member initialization
     this._entry = entry;
     this._contentType = contentType;
 }
Esempio n. 10
0
 public RemoteFile(S3Object s3Object)
     : base(s3Object.Key)
 {
     this.s3Obj = s3Object;
 }
Esempio n. 11
0
 public RemoteFile(string absolutePath)
     : base(absolutePath)
 {
     this.s3Obj = new S3Object();
 }
Esempio n. 12
0
 public S3StorageFolder(S3Object entry)
 {
     // TODO: Complete member initialization
     this._entry = entry;
 }
Esempio n. 13
0
        private String[] ReadS3File(S3Object fileName)
        {
            var gor = new GetObjectRequest().WithBucketName(BUCKET).WithKey(fileName.Key);
            var file = s3.GetObject(gor);
            string text = "";

            using (var ms = new MemoryStream())
            {
                Trace.WriteLine("Started reading file");
                file.ResponseStream.CopyTo(ms); //actually fetches the file from S3
                Trace.WriteLine("Finished reading file");
                using (var gzipStream = new GZipStream(new MemoryStream(ms.ToArray()), CompressionMode.Decompress))
                {
                    Trace.WriteLine("Decompressing file");
                    const int size = 4096;
                    byte[] buffer = new byte[size];
                    using (MemoryStream memory = new MemoryStream())
                    {
                        int count = 0;
                        do
                        {
                            count = gzipStream.Read(buffer, 0, size);
                            if (count > 0)
                            {
                                memory.Write(buffer, 0, count);
                            }
                        }
                        while (count > 0);
                        var memArray = memory.ToArray();
                        text = ASCIIEncoding.ASCII.GetString(memArray);
                    }
                    Trace.WriteLine("Finished decompressing file");
                }
            }

            var lines = text.Split(delimiters, StringSplitOptions.RemoveEmptyEntries);
            Trace.WriteLine("Finished reading file");
            return lines;
        }
Esempio n. 14
0
 private static Item Map(string root, S3Object src)
 {
     return new Item
     {
         Type = "S3Object",
         Path = src.Key,
         Length = src.Size,
         S3Object = src,
         Relative = src.Key.Substring(root.Length),
         Hash = src.ETag.Replace("\"", string.Empty)
     };
 }
Esempio n. 15
-1
        private static void DownloadFile(S3Object s3Object)
        {
            System.Console.WriteLine("Downloading " + s3Object.Key);
            GetObjectResponse getObjectResponse = _amazonS3Client.GetObject(new GetObjectRequest { BucketName = BucketName, Key = s3Object.Key });

            string filePath = Path.Combine(_folder, s3Object.Key);

            using (BufferedStream inputBufferedStream = new BufferedStream(getObjectResponse.ResponseStream))
            using (CryptoStream cryptoStream = new CryptoStream(inputBufferedStream, _aesDecryptor, CryptoStreamMode.Read))
            using (FileStream outputFileStream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite))
            {
                int data;
                while ((data = cryptoStream.ReadByte()) != -1)
                {
                    outputFileStream.WriteByte((byte)data);
                }
            }

            new FileInfo(filePath).LastWriteTime = DateTime.Parse(getObjectResponse.Metadata["x-amz-meta-LWT"]);
        }
        public AmazonFilesStatus(S3Object s3File,string bucket,string prefix)
        {
            var filename = s3File.Key.Replace(prefix,"");
            var fileExt = filename.Remove(0,filename.LastIndexOf('.'));
            type =  getContentTypeByExtension(fileExt);
            isimage =  Regex.Match(filename.ToLower(),AmazonHelper.ImgExtensions()).Success;
            var client = AmazonHelper.GetS3Client();
            var metareq = new GetObjectMetadataRequest().WithBucketName(bucket).WithKey(s3File.Key);
            var meta = client.GetObjectMetadata(metareq);
            var height = 0;
            var width = 0;
            if(isimage){
                height = String.IsNullOrWhiteSpace(meta.Headers["x-amz-meta-height"])? 0 : Int32.Parse(meta.Headers["x-amz-meta-height"]);
                 width = String.IsNullOrWhiteSpace(meta.Headers["x-amz-meta-width"])? 0 : Int32.Parse(meta.Headers["x-amz-meta-width"]);

            }

            var size = Convert.ToInt32(s3File.Size);

            SetValues(filename,bucket,prefix,size,height,width);
        }