Exemple #1
0
            /// <summary>
            /// Downloads a collection of files from S3 and writes ithem to the specified files.
            /// </summary>
            /// <param name="paths">The paths to upload.</param>
            ///  <param name="settings">The <see cref="SyncSettings"/> required to download from Amazon S3.</param>
            public void Download(IList<SyncPath> paths, SyncSettings settings)
            {
                foreach(SyncPath path in paths)
                {
                    try
                    {
                        DownloadSettings copied = new DownloadSettings()
                        {
                            WorkingDirectory = settings.WorkingDirectory,

                            AccessKey = settings.AccessKey,
                            SecretKey = settings.SecretKey,
                            Credentials = settings.Credentials,

                            Region = settings.Region,
                            BucketName = settings.BucketName,

                            EncryptionMethod = settings.EncryptionMethod,
                            EncryptionKey = settings.EncryptionKey,
                            EncryptionKeyMD5 = settings.EncryptionKeyMD5
                        };

                        this.Download(path.Path, path.Key, "", copied);
                    }
                    catch (Exception ex)
                    {
                        _Log.Error(ex.Message);
                    }
                }
            }
Exemple #2
0
            /// <summary>
            /// Syncs the specified file from Amazon S3, checking the modified date of the local files with existing S3Objects and downloading them if its changed.
            /// </summary>
            /// <param name="filePath">The file path to sync to S3</param>
            /// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
            /// <returns>The key that require invalidating.</returns>
            public string SyncDownload(FilePath filePath, SyncSettings settings)
            {
                //Get Directory
                this.SetWorkingDirectory(settings);
                string fullPath = filePath.MakeAbsolute(settings.WorkingDirectory).FullPath;
                if (!fullPath.EndsWith("/"))
                {
                    fullPath += "/";
                }

                IFile file = _FileSystem.GetFile(filePath.MakeAbsolute(settings.WorkingDirectory));

                if (settings.ModifiedCheck == ModifiedCheck.Hash)
                {
                    settings.GenerateETag = true;
                }
                
                string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
                S3Object obj = this.GetObject(key, "", settings);

                IList<SyncPath> download = new List<SyncPath>();



                if ((file.Exists) && (obj != null))
                {
                    //Get ETag
                    string eTag = "";
                    if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
                    {
                        eTag = this.GetHash(file);
                    }



                    //Check Modified
                    if (((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
                        || ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) < (DateTimeOffset)obj.LastModified))
                    {
                        download.Add(new SyncPath()
                        {
                            Path = file.Path,
                            Key = key
                        });
                    }
                }
                else if (obj != null)
                {
                    download.Add(new SyncPath()
                    {
                        Path = this.GetPath(fullPath, obj.Key),
                        Key = obj.Key
                    });
                }
                else
                {
                    //Delete
                    file.Delete();
                    
                    _Log.Verbose("Deleting file {0}", file.Path.FullPath);
                    return key;
                }



                //Download
                this.LogProgress = false;
                this.Download(download, settings);

                return (download.Count > 0) ? key : "";
            }
Exemple #3
0
            /// <summary>
            /// Uploads a collection of files to S3. For large uploads, the file will be divided and uploaded in parts 
            /// using Amazon S3's multipart API. The parts will be reassembled as one object in Amazon S3.
            /// </summary>
            /// <param name="paths">The paths to upload.</param>
            /// <param name="settings">The <see cref="SyncSettings"/> required to upload to Amazon S3.</param>
            public void Upload(IList<SyncPath> paths, SyncSettings settings)
            {
                foreach(SyncPath path in paths)
                {
                    try
                    {
                        UploadSettings copied = new UploadSettings()
                        {
                            WorkingDirectory = settings.WorkingDirectory,

                            AccessKey = settings.AccessKey,
                            SecretKey = settings.SecretKey,
                            Credentials = settings.Credentials,

                            Region = settings.Region,
                            BucketName = settings.BucketName,

                            EncryptionMethod = settings.EncryptionMethod,
                            EncryptionKey = settings.EncryptionKey,
                            EncryptionKeyMD5 = settings.EncryptionKeyMD5,

                            CannedACL = settings.CannedACL,
                            StorageClass = settings.StorageClass,

                            KeyManagementServiceKeyId = settings.KeyManagementServiceKeyId,

                            Headers = new HeadersCollection(),
                            GenerateContentType = settings.GenerateContentType,
                            GenerateETag = settings.GenerateETag,
                            GenerateHashTag = settings.GenerateHashTag
                        };

                        if (!String.IsNullOrEmpty(path.ETag))
                        {
                            copied.Headers["ETag"] = path.ETag;
                        }

                        this.Upload(path.Path, path.Key, copied);
                    }
                    catch (Exception ex)
                    {
                        _Log.Error(ex.Message);
                    }
                }
            }
Exemple #4
0
            /// <summary>
            /// Syncs the specified directory from Amazon S3, checking the modified date of the local files with existing S3Objects and downloading them if its changed.
            /// </summary>
            /// <param name="dirPath">The directory path to sync to S3</param>
            /// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
            /// <returns>A list of keys that require invalidating.</returns>
            public IList<string> SyncDownload(DirectoryPath dirPath, SyncSettings settings)
            {
                //Get Directory
                this.SetWorkingDirectory(settings);
                string fullPath = dirPath.MakeAbsolute(settings.WorkingDirectory).FullPath;
                if (!fullPath.EndsWith("/"))
                {
                    fullPath += "/";
                }

                IDirectory dir = _FileSystem.GetDirectory(dirPath.MakeAbsolute(settings.WorkingDirectory));
                List<string> list = new List<string>();

                if (settings.ModifiedCheck == ModifiedCheck.Hash)
                {
                    settings.GenerateETag = true;
                }



                if (dir.Exists)
                {
                    //Get S3 Objects
                    IList<S3Object> objects = this.GetObjects(settings.KeyPrefix, settings);
                    IEnumerable<IFile> files = dir.GetFiles(settings.SearchFilter, settings.SearchScope);

                    IList<SyncPath> download = new List<SyncPath>();

                    List<IFile> delete = new List<IFile>();
                    delete.AddRange(files);



                    foreach (S3Object obj in objects)
                    {
                        //Find File
                        IFile file = files.FirstOrDefault(f => 
                        {
                            return (this.GetKey(f, fullPath, settings.LowerPaths, settings.KeyPrefix) == obj.Key);
                        });

                        if (file != null)
                        {
                            //Get Key
                            string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);



                            //Get ETag
                            string eTag = "";
                            if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
                            {
                                eTag = this.GetHash(file);
                            }



                            //Check Modified
                            if (((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
                                || ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) < (DateTimeOffset)obj.LastModified))
                            {
                                download.Add(new SyncPath()
                                {
                                    Path = file.Path,
                                    Key = key
                                });

                                list.Add(key);
                            }

                            delete.Remove(file);
                        }
                        else
                        {
                            download.Add(new SyncPath()
                            {
                                Path = this.GetPath(fullPath, obj.Key),
                                Key = obj.Key
                            });

                            list.Add(obj.Key);
                        }
                    }



                    //Download
                    this.LogProgress = false;
                    this.Download(download, settings);



                    //Delete
                    foreach (IFile file in delete)
                    {
                        file.Delete();
                    
                        _Log.Verbose("Deleting file {0}", file.Path.FullPath);
                    }
                }

                return list;
            }
Exemple #5
0
            /// <summary>
            /// Syncs the specified file to Amazon S3, checking the modified date of the local files with existing S3Objects and uploading them if its changes.
            /// </summary>
            /// <param name="filePath">The file path to sync to S3</param>
            /// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
            /// <returns>The key that require invalidating.</returns>
            public string SyncUpload(FilePath filePath, SyncSettings settings)
            {
                //Get Directory
                this.SetWorkingDirectory(settings);
                string fullPath = filePath.MakeAbsolute(settings.WorkingDirectory).FullPath;

                IFile file = _FileSystem.GetFile(filePath.MakeAbsolute(settings.WorkingDirectory));

                if (settings.ModifiedCheck == ModifiedCheck.Hash)
                {
                    settings.GenerateETag = true;
                }

                string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
                S3Object obj = this.GetObject(key, "", settings);



                if (file.Exists)
                {
                    //Get ETag
                    string eTag = "";
                    if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
                    {
                        eTag = this.GetHash(file);
                    }



                    //Check Modified
                    IList<SyncPath> upload = new List<SyncPath>();

                    if ((obj == null)
                        || ((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
                        || ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) > (DateTimeOffset)obj.LastModified))
                    {
                        upload.Add(new SyncPath()
                        {
                            Path = file.Path,
                            Key = key,
                            ETag = eTag
                        });
                    }



                    //Upload
                    this.LogProgress = false;
                    this.Upload(upload, settings);

                    return key;
                }
                else if (obj != null)
                {
                    this.Delete(key, "", settings);

                    return key;
                }
                else
                {
                    return "";
                }
            }
Exemple #6
0
            /// <summary>
            /// Syncs the specified directory to Amazon S3, checking the modified date of the local files with existing S3Objects and uploading them if its changes.
            /// </summary>
            /// <param name="dirPath">The directory path to sync to S3</param>
            /// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
            /// <returns>A list of keys that require invalidating.</returns>
            public IList<string> SyncUpload(DirectoryPath dirPath, SyncSettings settings)
            {
                //Get Directory
                this.SetWorkingDirectory(settings);
                string fullPath = dirPath.MakeAbsolute(settings.WorkingDirectory).FullPath;
                if (!fullPath.EndsWith("/"))
                {
                    fullPath += "/";
                }

                IDirectory dir = _FileSystem.GetDirectory(dirPath.MakeAbsolute(settings.WorkingDirectory));
                List<string> list = new List<string>();

                if (settings.ModifiedCheck == ModifiedCheck.Hash)
                {
                    settings.GenerateETag = true;
                }



                if (dir.Exists)
                {
                    //Get S3 Objects
                    IList<S3Object> objects = this.GetObjects(settings.KeyPrefix, settings);



                    //Check Files
                    IEnumerable<IFile> files = dir.GetFiles(settings.SearchFilter, settings.SearchScope);
                    IList<SyncPath> upload = new List<SyncPath>();

                    foreach (IFile file in files)
                    {
                        //Get Key
                        string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);



                        //Get ETag
                        string eTag = "";
                        if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
                        {
                            eTag = this.GetHash(file);
                        }



                        //Check Modified
                        S3Object obj = objects.FirstOrDefault(o => o.Key == key);

                        if ((obj == null) 
                            || ((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
                            || ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) > (DateTimeOffset)obj.LastModified) )
                        {
                            upload.Add(new SyncPath()
                            {
                                Path = file.Path,
                                Key = key,
                                ETag = eTag
                            });

                            list.Add(key);
                        }

                        if (obj != null)
                        {
                            objects.Remove(obj);
                        }
                    }



                    //Upload
                    this.LogProgress = false;
                    this.Upload(upload, settings);



                    //Delete
                    list.AddRange(objects.Select(o => o.Key).ToList());

                    this.Delete(objects.Select(o => o.Key).ToList(), settings);
                }

                return list;
            }
Exemple #7
0
 public static IList <string> S3SyncDownload(this ICakeContext context, DirectoryPath dirPath, SyncSettings settings)
 {
     return(context.CreateManager().SyncDownload(dirPath, settings));
 }
Exemple #8
0
 public static string S3SyncDownload(this ICakeContext context, FilePath filePath, SyncSettings settings)
 {
     return(context.CreateManager().SyncDownload(filePath, settings));
 }
Exemple #9
0
 public static string S3SyncDownload(this ICakeContext context, FilePath filePath, SyncSettings settings)
 {
    return context.CreateManager().SyncDownload(filePath, settings);
 }
Exemple #10
0
 public static IList<string> S3SyncUpload(this ICakeContext context, DirectoryPath dirPath, SyncSettings settings)
 {
    return context.CreateManager().SyncUpload(dirPath, settings);
 }