/// <summary>Remove S3 file from cache local storage.</summary>
/// <param name="obj">S3 object to be removed from cache local storage</param>
private static void RemoveFromCache(IS3ObjectInfo obj)
{
string path = CMS.IO.Path.Combine(PathHelper.CachePath, PathHelper.GetPathFromObjectKey(obj.Key, false));
DeleteFileFromLocalPath(path);
DeleteFileFromLocalPath(path + ".etag");
}
/// <summary>Appends text to Amazon S3 storage object.</summary>
/// <param name="obj">Object info.</param>
/// <param name="content">Content to append.</param>
public void AppendTextToObject(IS3ObjectInfo obj, string content)
{
if (this.ObjectExists(obj))
{
if (obj.IsLocked)
{
throw new Exception($"[IS3ObjectInfoProvider.AppendTextToObject]: Couldn't upload object {obj.Key} because it is used by another process.");
}
obj.Lock();
System.IO.Stream objectContent = this.GetObjectContent(obj, System.IO.FileMode.Open,
System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite, 4096);
string str = null;
using (CMS.IO.StreamReader streamReader = CMS.IO.StreamReader.New(objectContent))
{
str = streamReader.ReadToEnd();
}
PutObjectRequest putRequest = CreatePutRequest(obj.Key, obj.BucketName);
putRequest.ContentBody = str + content;
PutObjectResponse response = this.S3Client.PutObject(putRequest);
this.SetS3ObjectMetadaFromResponse(obj, response, 0L);
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), nameof(AppendTextToObject), "Custom Amazon");
obj.UnLock();
RemoveRequestCache(obj.Key);
}
else
{
this.PutTextToObject(obj, content);
}
}
/// <summary>Puts data from stream to Amazon S3 storage.</summary>
/// <param name="obj">Object info.</param>
/// <param name="stream">Stream to upload.</param>
public void PutDataFromStreamToObject(IS3ObjectInfo obj, System.IO.Stream stream)
{
if (obj.IsLocked)
{
throw new Exception($"[IS3ObjectInfoProvider.PutDataFromStreamToObject]: Couldn't upload object {obj.Key} because it is used by another process.");
}
obj.Lock();
string bucketName = obj.BucketName;
long length = stream.Length;
if (length > RECOMMENDED_SIZE_FOR_MULTIPART_UPLOAD)
{
CompleteMultipartUploadResponse response = this.MultiPartUploader.UploadFromStream(obj.Key, bucketName, stream);
this.SetS3ObjectMetadaFromResponse(obj, response, length);
}
else
{
PutObjectRequest putRequest = CreatePutRequest(obj.Key, bucketName);
putRequest.InputStream = stream;
PutObjectResponse response = this.S3Client.PutObject(putRequest);
this.SetS3ObjectMetadaFromResponse(obj, response, length);
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), "PutStreamToObject", "Custom Amazon");
obj.UnLock();
RemoveRequestCache(obj.Key);
}
/// <summary>Unlocks current object.</summary>
public void UnLock()
{
if (!this.Provider.ObjectExists(this))
{
return;
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(this.Key, true), "Unlock", "Custom Amazon");
this.SetMetadata(S3ObjectInfoProvider.LOCK, "False", true, false);
}
/// <summary>Locks current object.</summary>
public void Lock()
{
if (!this.Provider.ObjectExists(this))
{
return;
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(this.Key, true), nameof(Lock), "Custom Amazon");
this.SetMetadata(nameof(Lock), "True", true, false);
}
/// <summary>Creates empty object.</summary>
/// <param name="obj">Object info.</param>
public void CreateEmptyObject(IS3ObjectInfo obj)
{
string pathFromObjectKey = PathHelper.GetPathFromObjectKey(obj.Key, true);
PutObjectRequest putRequest = CreatePutRequest(obj.Key, GetBucketName(pathFromObjectKey));
putRequest.InputStream = new System.IO.MemoryStream();
PutObjectResponse response = this.S3Client.PutObject(putRequest);
this.SetS3ObjectMetadaFromResponse(obj, response, 0L);
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), nameof(CreateEmptyObject), "Custom Amazon");
RemoveRequestCache(obj.Key);
}
/// <summary>Returns object content as a stream.</summary>
/// <param name="obj">Object info.</param>
/// <param name="fileMode">File mode.</param>
/// <param name="fileAccess">File access.</param>
/// <param name="fileShare">Sharing permissions.</param>
/// <param name="bufferSize">Buffer size.</param>
public System.IO.Stream GetObjectContent(IS3ObjectInfo obj, System.IO.FileMode fileMode = System.IO.FileMode.Open,
System.IO.FileAccess fileAccess = System.IO.FileAccess.Read, System.IO.FileShare fileShare = System.IO.FileShare.ReadWrite, int bufferSize = 4096)
{
if (!this.ObjectExists(obj))
{
return(null);
}
AutoResetEvent orAdd = mS3ObjectEvents.GetOrAdd(obj.Key, new AutoResetEvent(true));
try
{
string tempPath = CMS.IO.Path.Combine(PathHelper.TempPath, PathHelper.GetPathFromObjectKey(obj.Key, false));
Directory.CreateDiskDirectoryStructure(tempPath);
string cachePath = CMS.IO.Path.Combine(PathHelper.CachePath, PathHelper.GetPathFromObjectKey(obj.Key, false));
string path = $"{cachePath}.etag";
orAdd.WaitOne();
if (CMS.IO.File.Exists(cachePath) && System.IO.File.ReadAllText(path).Trim() == obj.ETag)
{
orAdd.Set();
System.IO.FileStream fileStream = new System.IO.FileStream(cachePath, fileMode, fileAccess, fileShare, bufferSize);
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), "GetObjectFromCache", "Custom Amazon");
return(fileStream);
}
using (GetObjectResponse getObjectResponse = this.S3Client.GetObject(new GetObjectRequest()
{
BucketName = obj.BucketName,
Key = obj.Key
}))
{
getObjectResponse.WriteResponseStreamToFile(tempPath);
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), "GetObjectFromS3Storage", "Custom Amazon");
Directory.CreateDiskDirectoryStructure(cachePath);
System.IO.File.Copy(tempPath, cachePath, true);
System.IO.File.WriteAllText(path, obj.ETag);
if (fileMode == System.IO.FileMode.Append && fileAccess != System.IO.FileAccess.Read)
{
fileMode = System.IO.FileMode.Open;
fileAccess = System.IO.FileAccess.ReadWrite;
}
return(new System.IO.FileStream(tempPath, fileMode, fileAccess, fileShare, bufferSize));
}
finally
{
orAdd.Set();
mS3ObjectEvents.TryRemove(obj.Key, out orAdd);
}
}
/// <summary>Deletes object from Amazon S3 storage.</summary>
/// <param name="obj">Object info.</param>
public void DeleteObject(IS3ObjectInfo obj)
{
this.S3Client.DeleteObject(new DeleteObjectRequest()
{
BucketName = obj.BucketName,
Key = obj.Key
});
obj.DeleteMetadataFile();
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), nameof(DeleteObject), "Custom Amazon");
RemoveRequestCache(obj.Key);
try
{
RemoveFromTemp(obj);
RemoveFromCache(obj);
}
catch (IOException)
{
}
}
/// <summary>Puts text to Amazon S3 storage object.</summary>
/// <param name="obj">Object info.</param>
/// <param name="content">Content to add.</param>
public void PutTextToObject(IS3ObjectInfo obj, string content)
{
if (obj.IsLocked)
{
throw new Exception($"[IS3ObjectInfoProvider.PutTextToObject]: Couldn't upload object {obj.Key} because it is used by another process.");
}
string pathFromObjectKey = PathHelper.GetPathFromObjectKey(obj.Key, true);
obj.Lock();
PutObjectRequest putRequest = CreatePutRequest(obj.Key, GetBucketName(pathFromObjectKey));
putRequest.ContentBody = content;
PutObjectResponse response = this.S3Client.PutObject(putRequest);
this.SetS3ObjectMetadaFromResponse(obj, response, 0L);
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), nameof(PutTextToObject), "Custom Amazon");
obj.UnLock();
RemoveRequestCache(obj.Key);
}
/// <summary>Sets meta data to object.</summary>
/// <param name="key">MetaData key.</param>
/// <param name="value">Metadata value.</param>
/// <param name="update">Indicates whether data are updated in S3 storage.</param>
/// <param name="log">Indicates whether is operation logged.</param>
public void SetMetadata(string key, string value, bool update, bool log)
{
if (this.Metadata.ContainsKey(key))
{
this.Metadata[key] = value;
}
else
{
this.Metadata.Add(key, value);
}
if (update)
{
this.SaveMetadata(PathHelper.GetPathFromObjectKey(this.Key, true), this.Metadata);
}
if (!log)
{
return;
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(this.Key, true), nameof(SetMetadata), "Custom Amazon");
}
/// <summary>
/// Downloads metadata from the cloud. It ensures that RequestStockHelper always contains the "Exists" key.
/// </summary>
private void FetchMetadata()
{
if (AbstractStockHelper <RequestStockHelper> .Contains(STORAGE_KEY, this.Key + "|Exists", false))
{
return;
}
GetObjectMetadataRequest request = new GetObjectMetadataRequest
{
BucketName = this.BucketName,
Key = this.Key
};
try
{
GetObjectMetadataResponse objectMetadata = this.S3Client.GetObjectMetadata(request);
AbstractStockHelper <RequestStockHelper> .AddToStorage(STORAGE_KEY, this.Key + "|Length", objectMetadata.ContentLength, false);
AbstractStockHelper <RequestStockHelper> .AddToStorage(STORAGE_KEY, this.Key + "|ETag", objectMetadata.ETag, false);
AbstractStockHelper <RequestStockHelper> .AddToStorage(STORAGE_KEY, this.Key + "|Exists", true, false);
this.Metadata = this.LoadMetadata(PathHelper.GetPathFromObjectKey(this.Key, true));
if (!this.Metadata.ContainsKey(S3ObjectInfoProvider.LAST_WRITE_TIME))
{
this.Metadata.Add(S3ObjectInfoProvider.LAST_WRITE_TIME, ValidationHelper.GetString(objectMetadata.LastModified, string.Empty, "en-us"));
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(this.Key, true), nameof(FetchMetadata), "Custom Amazon");
}
catch (AmazonS3Exception ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
AbstractStockHelper <RequestStockHelper> .AddToStorage(STORAGE_KEY, this.Key + "|Exists", false, false);
}
else
{
throw;
}
}
}
/// <summary>Copies object to another.</summary>
/// <param name="sourceObject">Source object info.</param>
/// <param name="destObject">Destination object info.</param>
public void CopyObjects(IS3ObjectInfo sourceObject, IS3ObjectInfo destObject)
{
string pathFromObjectKey = PathHelper.GetPathFromObjectKey(destObject.Key, true);
CopyObjectRequest request = new CopyObjectRequest()
{
SourceBucket = sourceObject.BucketName,
DestinationBucket = GetBucketName(pathFromObjectKey),
SourceKey = sourceObject.Key,
DestinationKey = destObject.Key
};
if (this.IsPublicAccess(pathFromObjectKey))
{
request.CannedACL = S3CannedACL.PublicRead;
}
CopyObjectResponse copyObjectResponse = this.S3Client.CopyObject(request);
destObject.ETag = copyObjectResponse.ETag;
destObject.Length = ValidationHelper.GetLong(copyObjectResponse.ContentLength, 0L);
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(sourceObject.Key, true) + "|" + PathHelper.GetPathFromObjectKey(destObject.Key, true), nameof(CopyObjects), "Custom Amazon");
RemoveRequestCache(destObject.Key);
}
/// <summary>Remove S3 file from temporary local storage.</summary>
/// <param name="obj">S3 object to be removed from temporary local storage</param>
private static void RemoveFromTemp(IS3ObjectInfo obj)
{
DeleteFileFromLocalPath(CMS.IO.Path.Combine(PathHelper.TempPath, PathHelper.GetPathFromObjectKey(obj.Key, false)));
}
/// <summary>
/// Returns list with objects from given bucket and under given path.
/// </summary>
/// <param name="path">Path.</param>
/// <param name="type">Specifies which objects are returned (files, directories, both).</param>
/// <param name="useFlatListing">Whether flat listing is used (all files from all subdirectories all in the result).</param>
/// <param name="lower">Specifies whether path should be lowered inside method.</param>
/// <param name="useCache">Indicates if results should be primary taken from cache to get better performance</param>
/// <remarks>
/// In order to allow to distinguish between files and directories, directories are listed with a trailing backslash.
/// </remarks>
public List <string> GetObjectsList(string path, ObjectTypeEnum type, bool useFlatListing = false, bool lower = true, bool useCache = true)
{
string bucketName = GetBucketName(path);
if (string.IsNullOrEmpty(bucketName))
{
return(null);
}
var request = new ListObjectsRequest
{
BucketName = bucketName
};
if (!string.IsNullOrEmpty(path))
{
request.Prefix = PathHelper.GetObjectKeyFromPath(path).TrimEnd('/') + "/";
}
if (!useFlatListing)
{
request.Delimiter = "/";
}
var source1 = new HashSet <string>();
string key = $"{request.Prefix }|{type.ToString("F")}|{useFlatListing}|{lower}";
if (useCache && AbstractStockHelper <RequestStockHelper> .Contains(STORAGE_KEY, key, false))
{
CMS.IO.Directory.LogDirectoryOperation(path, "GetObjectListFromCache", "Custom Amazon");
var source2 = AbstractStockHelper <RequestStockHelper> .GetItem(STORAGE_KEY, key, false) as HashSet <string>;
if (source2 == null)
{
return(new List <string>());
}
return(source2.ToList());
}
ListObjectsResponse listObjectsResponse;
do
{
listObjectsResponse = this.S3Client.ListObjects(request);
if (type == ObjectTypeEnum.Directories && !useFlatListing)
{
foreach (string commonPrefix in listObjectsResponse.CommonPrefixes)
{
source1.Add($"{path}\\{CMS.IO.Path.GetFileName(commonPrefix.TrimEnd('/'))}\\");
}
}
else
{
bool isDirectory = type == ObjectTypeEnum.FilesAndDirectories || type == ObjectTypeEnum.Directories;
bool isFile = type == ObjectTypeEnum.FilesAndDirectories || type == ObjectTypeEnum.Files;
foreach (S3Object s3Object in listObjectsResponse.S3Objects)
{
bool directory = s3Object.Key.EndsWith("/", StringComparison.Ordinal);
if (directory && isDirectory || !directory && isFile)
{
source1.Add(PathHelper.GetPathFromObjectKey(s3Object.Key, true, directory, lower));
}
}
}
if (listObjectsResponse.IsTruncated)
{
request.Marker = listObjectsResponse.NextMarker;
}
}while (listObjectsResponse.IsTruncated);
source1.Remove(PathHelper.GetPathFromObjectKey(request.Prefix, true, true, lower));
CMS.IO.Directory.LogDirectoryOperation(path, "ListObjects", "Custom Amazon");
AbstractStockHelper <RequestStockHelper> .AddToStorage(STORAGE_KEY, key, (object)source1, false);
return(source1.ToList());
}
