/// <summary>
/// Downloads a collection of files from S3 and writes ithem to the specified files.
/// </summary>
/// <param name="paths">The paths to upload.</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to download from Amazon S3.</param>
public void Download(IList<SyncPath> paths, SyncSettings settings)
{
foreach(SyncPath path in paths)
{
try
{
DownloadSettings copied = new DownloadSettings()
{
WorkingDirectory = settings.WorkingDirectory,
AccessKey = settings.AccessKey,
SecretKey = settings.SecretKey,
Credentials = settings.Credentials,
Region = settings.Region,
BucketName = settings.BucketName,
EncryptionMethod = settings.EncryptionMethod,
EncryptionKey = settings.EncryptionKey,
EncryptionKeyMD5 = settings.EncryptionKeyMD5
};
this.Download(path.Path, path.Key, "", copied);
}
catch (Exception ex)
{
_Log.Error(ex.Message);
}
}
}
/// <summary>
/// Syncs the specified file from Amazon S3, checking the modified date of the local files with existing S3Objects and downloading them if its changed.
/// </summary>
/// <param name="filePath">The file path to sync to S3</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
/// <returns>The key that require invalidating.</returns>
public string SyncDownload(FilePath filePath, SyncSettings settings)
{
//Get Directory
this.SetWorkingDirectory(settings);
string fullPath = filePath.MakeAbsolute(settings.WorkingDirectory).FullPath;
if (!fullPath.EndsWith("/"))
{
fullPath += "/";
}
IFile file = _FileSystem.GetFile(filePath.MakeAbsolute(settings.WorkingDirectory));
if (settings.ModifiedCheck == ModifiedCheck.Hash)
{
settings.GenerateETag = true;
}
string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
S3Object obj = this.GetObject(key, "", settings);
IList<SyncPath> download = new List<SyncPath>();
if ((file.Exists) && (obj != null))
{
//Get ETag
string eTag = "";
if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
{
eTag = this.GetHash(file);
}
//Check Modified
if (((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
|| ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) < (DateTimeOffset)obj.LastModified))
{
download.Add(new SyncPath()
{
Path = file.Path,
Key = key
});
}
}
else if (obj != null)
{
download.Add(new SyncPath()
{
Path = this.GetPath(fullPath, obj.Key),
Key = obj.Key
});
}
else
{
//Delete
file.Delete();
_Log.Verbose("Deleting file {0}", file.Path.FullPath);
return key;
}
//Download
this.LogProgress = false;
this.Download(download, settings);
return (download.Count > 0) ? key : "";
}
/// <summary>
/// Uploads a collection of files to S3. For large uploads, the file will be divided and uploaded in parts
/// using Amazon S3's multipart API. The parts will be reassembled as one object in Amazon S3.
/// </summary>
/// <param name="paths">The paths to upload.</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to upload to Amazon S3.</param>
public void Upload(IList<SyncPath> paths, SyncSettings settings)
{
foreach(SyncPath path in paths)
{
try
{
UploadSettings copied = new UploadSettings()
{
WorkingDirectory = settings.WorkingDirectory,
AccessKey = settings.AccessKey,
SecretKey = settings.SecretKey,
Credentials = settings.Credentials,
Region = settings.Region,
BucketName = settings.BucketName,
EncryptionMethod = settings.EncryptionMethod,
EncryptionKey = settings.EncryptionKey,
EncryptionKeyMD5 = settings.EncryptionKeyMD5,
CannedACL = settings.CannedACL,
StorageClass = settings.StorageClass,
KeyManagementServiceKeyId = settings.KeyManagementServiceKeyId,
Headers = new HeadersCollection(),
GenerateContentType = settings.GenerateContentType,
GenerateETag = settings.GenerateETag,
GenerateHashTag = settings.GenerateHashTag
};
if (!String.IsNullOrEmpty(path.ETag))
{
copied.Headers["ETag"] = path.ETag;
}
this.Upload(path.Path, path.Key, copied);
}
catch (Exception ex)
{
_Log.Error(ex.Message);
}
}
}
/// <summary>
/// Syncs the specified directory from Amazon S3, checking the modified date of the local files with existing S3Objects and downloading them if its changed.
/// </summary>
/// <param name="dirPath">The directory path to sync to S3</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
/// <returns>A list of keys that require invalidating.</returns>
public IList<string> SyncDownload(DirectoryPath dirPath, SyncSettings settings)
{
//Get Directory
this.SetWorkingDirectory(settings);
string fullPath = dirPath.MakeAbsolute(settings.WorkingDirectory).FullPath;
if (!fullPath.EndsWith("/"))
{
fullPath += "/";
}
IDirectory dir = _FileSystem.GetDirectory(dirPath.MakeAbsolute(settings.WorkingDirectory));
List<string> list = new List<string>();
if (settings.ModifiedCheck == ModifiedCheck.Hash)
{
settings.GenerateETag = true;
}
if (dir.Exists)
{
//Get S3 Objects
IList<S3Object> objects = this.GetObjects(settings.KeyPrefix, settings);
IEnumerable<IFile> files = dir.GetFiles(settings.SearchFilter, settings.SearchScope);
IList<SyncPath> download = new List<SyncPath>();
List<IFile> delete = new List<IFile>();
delete.AddRange(files);
foreach (S3Object obj in objects)
{
//Find File
IFile file = files.FirstOrDefault(f =>
{
return (this.GetKey(f, fullPath, settings.LowerPaths, settings.KeyPrefix) == obj.Key);
});
if (file != null)
{
//Get Key
string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
//Get ETag
string eTag = "";
if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
{
eTag = this.GetHash(file);
}
//Check Modified
if (((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
|| ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) < (DateTimeOffset)obj.LastModified))
{
download.Add(new SyncPath()
{
Path = file.Path,
Key = key
});
list.Add(key);
}
delete.Remove(file);
}
else
{
download.Add(new SyncPath()
{
Path = this.GetPath(fullPath, obj.Key),
Key = obj.Key
});
list.Add(obj.Key);
}
}
//Download
this.LogProgress = false;
this.Download(download, settings);
//Delete
foreach (IFile file in delete)
{
file.Delete();
_Log.Verbose("Deleting file {0}", file.Path.FullPath);
}
}
return list;
}
/// <summary>
/// Syncs the specified file to Amazon S3, checking the modified date of the local files with existing S3Objects and uploading them if its changes.
/// </summary>
/// <param name="filePath">The file path to sync to S3</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
/// <returns>The key that require invalidating.</returns>
public string SyncUpload(FilePath filePath, SyncSettings settings)
{
//Get Directory
this.SetWorkingDirectory(settings);
string fullPath = filePath.MakeAbsolute(settings.WorkingDirectory).FullPath;
IFile file = _FileSystem.GetFile(filePath.MakeAbsolute(settings.WorkingDirectory));
if (settings.ModifiedCheck == ModifiedCheck.Hash)
{
settings.GenerateETag = true;
}
string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
S3Object obj = this.GetObject(key, "", settings);
if (file.Exists)
{
//Get ETag
string eTag = "";
if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
{
eTag = this.GetHash(file);
}
//Check Modified
IList<SyncPath> upload = new List<SyncPath>();
if ((obj == null)
|| ((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
|| ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) > (DateTimeOffset)obj.LastModified))
{
upload.Add(new SyncPath()
{
Path = file.Path,
Key = key,
ETag = eTag
});
}
//Upload
this.LogProgress = false;
this.Upload(upload, settings);
return key;
}
else if (obj != null)
{
this.Delete(key, "", settings);
return key;
}
else
{
return "";
}
}
/// <summary>
/// Syncs the specified directory to Amazon S3, checking the modified date of the local files with existing S3Objects and uploading them if its changes.
/// </summary>
/// <param name="dirPath">The directory path to sync to S3</param>
/// <param name="settings">The <see cref="SyncSettings"/> required to sync to Amazon S3.</param>
/// <returns>A list of keys that require invalidating.</returns>
public IList<string> SyncUpload(DirectoryPath dirPath, SyncSettings settings)
{
//Get Directory
this.SetWorkingDirectory(settings);
string fullPath = dirPath.MakeAbsolute(settings.WorkingDirectory).FullPath;
if (!fullPath.EndsWith("/"))
{
fullPath += "/";
}
IDirectory dir = _FileSystem.GetDirectory(dirPath.MakeAbsolute(settings.WorkingDirectory));
List<string> list = new List<string>();
if (settings.ModifiedCheck == ModifiedCheck.Hash)
{
settings.GenerateETag = true;
}
if (dir.Exists)
{
//Get S3 Objects
IList<S3Object> objects = this.GetObjects(settings.KeyPrefix, settings);
//Check Files
IEnumerable<IFile> files = dir.GetFiles(settings.SearchFilter, settings.SearchScope);
IList<SyncPath> upload = new List<SyncPath>();
foreach (IFile file in files)
{
//Get Key
string key = this.GetKey(file, fullPath, settings.LowerPaths, settings.KeyPrefix);
//Get ETag
string eTag = "";
if (settings.GenerateETag || (settings.ModifiedCheck == ModifiedCheck.Hash))
{
eTag = this.GetHash(file);
}
//Check Modified
S3Object obj = objects.FirstOrDefault(o => o.Key == key);
if ((obj == null)
|| ((settings.ModifiedCheck == ModifiedCheck.Hash) && (obj.ETag != "\"" + eTag + "\""))
|| ((settings.ModifiedCheck == ModifiedCheck.Date) && ((DateTimeOffset)new FileInfo(file.Path.FullPath).LastWriteTime) > (DateTimeOffset)obj.LastModified) )
{
upload.Add(new SyncPath()
{
Path = file.Path,
Key = key,
ETag = eTag
});
list.Add(key);
}
if (obj != null)
{
objects.Remove(obj);
}
}
//Upload
this.LogProgress = false;
this.Upload(upload, settings);
//Delete
list.AddRange(objects.Select(o => o.Key).ToList());
this.Delete(objects.Select(o => o.Key).ToList(), settings);
}
return list;
}
public static IList <string> S3SyncDownload(this ICakeContext context, DirectoryPath dirPath, SyncSettings settings)
{
return(context.CreateManager().SyncDownload(dirPath, settings));
}
public static string S3SyncDownload(this ICakeContext context, FilePath filePath, SyncSettings settings)
{
return(context.CreateManager().SyncDownload(filePath, settings));
}
public static string S3SyncDownload(this ICakeContext context, FilePath filePath, SyncSettings settings)
{
return context.CreateManager().SyncDownload(filePath, settings);
}
public static IList<string> S3SyncUpload(this ICakeContext context, DirectoryPath dirPath, SyncSettings settings)
{
return context.CreateManager().SyncUpload(dirPath, settings);
}