public void LoadFromStream(System.IO.Stream stream)
{
using (var fileTransferUtility = new Amazon.S3.Transfer.TransferUtility(_amazonS3))
{
fileTransferUtility.Upload(stream, _bucketName, _keyName);
}
}
public async System.Threading.Tasks.Task <SoftmakeAll.SDK.OperationResult <System.Text.Json.JsonElement> > UploadAsync(System.String BucketName, System.String StorageFileName, System.IO.Stream FileContents)
{
SoftmakeAll.SDK.CloudStorage.AWS.Environment.Validate();
SoftmakeAll.SDK.OperationResult <System.Text.Json.JsonElement> OperationResult = new SoftmakeAll.SDK.OperationResult <System.Text.Json.JsonElement>();
if (SoftmakeAll.SDK.Helpers.String.Extensions.StringExtensions.IsNullOrWhiteSpace(BucketName, StorageFileName))
{
OperationResult.Message = "The BucketName and StorageFileName cannot be null.";
return(OperationResult);
}
try
{
using (Amazon.S3.Transfer.TransferUtility TransferUtility = new Amazon.S3.Transfer.TransferUtility(SoftmakeAll.SDK.CloudStorage.AWS.Environment._S3Client))
await TransferUtility.UploadAsync(FileContents, BucketName, StorageFileName);
}
catch (System.Exception ex)
{
OperationResult.Message = ex.Message;
return(OperationResult);
}
OperationResult.ExitCode = 0;
return(OperationResult);
}
private bool WriteRawFileToS3(string strContent, string jobName)
{
byte[] memstring = Encoding.UTF8.GetBytes(strContent);
using (Stream memStream = new MemoryStream(strContent.Length))
{
memStream.Write(memstring, 0, memstring.Count());
// upload to s3
try
{
AmazonS3Client s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast2);
using (Amazon.S3.Transfer.TransferUtility tranUtility =
new Amazon.S3.Transfer.TransferUtility(s3))
{
tranUtility.Upload(memStream, "raw-transcripts", jobName + ".json");
return(true);
}
}
catch (Exception ex)
{
throw ex;
}
}
}
public S3Upload()
{
// Initialize log4net.
log4net.Config.XmlConfigurator.Configure();
this.transferUtility = new Amazon.S3.Transfer.TransferUtility(awsAccessKeyId, awsSecretAccessKey);
log.Info("S3 instance initiated");
}
public void SimpleUploadWithRequest()
{
RunAsSync(async() =>
{
try
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
string key = System.Guid.NewGuid().ToString();
Amazon.S3.Transfer.TransferUtilityUploadRequest request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
FilePath = fullPath,
StorageClass = S3StorageClass.ReducedRedundancy,
PartSize = 6291456, // 6 MB.
Key = key,
CannedACL = S3CannedACL.PublicRead
};
Stream filestream = this.BaseFolder.GetFileAsync(fullPath).Result.OpenAsync(FileAccess.Read).Result;
await tu.UploadAsync(fullPath, bucketName, key);
await VerifyUploadResponse(client, key);
await TestDownloadedFile(fullPath, tu, key);
}
}
catch (Exception e)
{
throw e;
}
});
}
private bool TransformFileToS3(string strContent, string jobName)
{
JObject transcriptJSON = JObject.Parse(strContent);
JObject sendObject = new JObject(
new JProperty("JobName", (string)transcriptJSON["jobName"]),
new JProperty("Text", (string)transcriptJSON["results"]["transcripts"][0]["transcript"]));
// Create file in memory
byte[] memstring = Encoding.UTF8.GetBytes(sendObject.ToString());
using (Stream memStream = new MemoryStream(strContent.Length))
{
memStream.Write(memstring, 0, memstring.Count());
// upload to s3
try
{
AmazonS3Client s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast2);
using (Amazon.S3.Transfer.TransferUtility tranUtility =
new Amazon.S3.Transfer.TransferUtility(s3))
{
tranUtility.Upload(memStream, "transcripts", jobName + ".json");
return(true);
}
}
catch (Exception ex)
{
throw ex;
}
}
}
public string UploadImageToS3(AST.Address addr, Bitmap b, string secret)
{
// convert Bitmap to MemoryStream
MemoryStream stream = new MemoryStream();
b.Save(stream, System.Drawing.Imaging.ImageFormat.Png);
// the image name is the md
var imagename = GetImageName(addr);
// the url to the bitmap
string url;
// upload MemoryStream to S3
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(_id, secret))
{
// generate url
GetPreSignedUrlRequest request = new GetPreSignedUrlRequest()
{
BucketName = _s3bucket,
Key = imagename,
Verb = HttpVerb.GET,
Expires = DateTime.Now.AddMonths(24)
};
url = client.GetPreSignedURL(request);
// upload image
var tu = new Amazon.S3.Transfer.TransferUtility(client);
tu.Upload(stream, _s3bucket, imagename);
}
return(url);
}
public override Uri UploadWithoutQuota(string domain, string path, Stream stream, string contentType, string contentDisposition)
{
ACL acl = ACL.Auto;
using (AmazonS3 client = GetClient())
{
var util = new Amazon.S3.Transfer.TransferUtility(client);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest();
string mime = string.IsNullOrEmpty(contentType)
? MimeMapping.GetMimeMapping(Path.GetFileName(path))
: contentType;
const int uploadTimeout = 10 * 60 * 1000;
request.BucketName = _bucket;
request.Key = MakePath(domain, path);
request.CannedACL = acl == ACL.Auto ? GetDomainACL(domain) : GetS3Acl(acl);
request.ContentType = mime;
request.Timeout = uploadTimeout;
var headers = new NameValueCollection();
headers.Add("Cache-Control", string.Format("public, maxage={0}", (int)TimeSpan.FromDays(5).TotalSeconds));
headers.Add("Etag", (DateTime.UtcNow.Ticks).ToString(CultureInfo.InvariantCulture));
headers.Add("Last-Modified", DateTime.UtcNow.ToString("R"));
headers.Add("Expires", DateTime.UtcNow.Add(TimeSpan.FromDays(5)).ToString("R"));
if (!string.IsNullOrEmpty(contentDisposition))
{
headers.Add("Content-Disposition", Uri.EscapeDataString(contentDisposition));
}
else if (mime == "application/octet-stream")
{
headers.Add("Content-Disposition", "attachment");
}
request.AddHeaders(headers);
//Send body
var buffered = stream.GetBuffered();
request.AutoCloseStream = false;
request.InputStream = buffered;
util.Upload(request);
InvalidateCloudFront(MakePath(domain, path));
return(GetUri(domain, path));
}
}
void ICoreAmazonS3.UploadObjectFromFilePath(string bucketName, string objectKey, string filepath, IDictionary <string, object> additionalProperties)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
transfer.Upload(request);
}
Task ICoreAmazonS3.UploadObjectFromStreamAsync(string bucketName, string objectKey, Stream stream, IDictionary <string, object> additionalProperties, CancellationToken cancellationToken)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
InputStream = stream
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return(transfer.UploadAsync(request, cancellationToken));
}
public void SimpleUploadWithFilepath()
{
RunAsSync(async() =>
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
await tu.UploadAsync(fullPath, bucketName);
await VerifyUploadResponse(client);
await TestDownloadedFile(fullPath, tu);
}
});
}
void ICoreAmazonS3.UploadObjectFromStream(string bucketName, string objectKey, Stream stream, IDictionary <string, object> additionalProperties)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
InputStream = stream
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
transfer.Upload(request);
}
IAsyncResult ICoreAmazonS3.BeginUploadObjectFromFilePath(string bucketName, string objectKey, string filepath, IDictionary <string, object> additionalProperties, AsyncCallback callback, object state)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return(transfer.BeginUpload(request, callback, state));
}
Task ICoreAmazonS3.DownloadToFilePathAsync(string bucketName, string objectKey, string filepath, IDictionary <string, object> additionalProperties, CancellationToken cancellationToken)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return(transfer.DownloadAsync(request, cancellationToken));
}
public bool SaveFile(string sFolder, string sObjectKey, byte[] fileContent, bool bMakePublic)
{
try
{
AmazonS3 client = AWSClientFactory.CreateAmazonS3Client(S3ACCESSKEY, S3SECRETKEY);
Amazon.S3.Transfer.TransferUtility uploader = new Amazon.S3.Transfer.TransferUtility(S3ACCESSKEY, S3SECRETKEY);
string BUCKET_NAME = ConfigurationManager.AppSettings["AWSBUCKET"];
ListBucketsResponse response = client.ListBuckets();
bool found = false;
foreach (S3Bucket bucket in response.Buckets)
{
if (bucket.BucketName == BUCKET_NAME)
{
found = true;
break;
}
}
if (found == false)
{
client.PutBucket(new PutBucketRequest().WithBucketName(BUCKET_NAME));
}
if (sFolder != "")
{
sObjectKey = sFolder + "/" + sObjectKey;
}
System.IO.Stream stream = new System.IO.MemoryStream(fileContent);
Amazon.S3.Transfer.TransferUtilityUploadRequest request = new Amazon.S3.Transfer.TransferUtilityUploadRequest();
request.WithBucketName(BUCKET_NAME);
request.WithKey(sObjectKey);
request.WithInputStream(stream);
request.WithTimeout(-1);
if (bMakePublic)
{
request.CannedACL = S3CannedACL.PublicRead;
}
uploader.Upload(request);
return(true);
}
catch (Exception)
{
return(false);
}
}
private static bool SendFileToS3(MemoryStream memStream, string bucket, string key)
{
try
{
AmazonS3Client s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast2);
using (Amazon.S3.Transfer.TransferUtility tranUtility =
new Amazon.S3.Transfer.TransferUtility(s3))
{
tranUtility.Upload(memStream, bucket, key);
return(true);
}
}
catch (Exception ex)
{
return(false);
}
}
public void SimpleUploadWithFilepathAndKey()
{
RunAsSync(async() =>
{
try
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
string key = System.Guid.NewGuid().ToString();
await tu.UploadAsync(fullPath, bucketName, key);
await VerifyUploadResponse(client, key);
await TestDownloadedFile(fullPath, tu, key);
}
}
catch (Exception e)
{
throw e;
}
});
}
public void SimpleUploadWithStream()
{
RunAsSync(async() =>
{
try
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
Stream filestream = this.BaseFolder.GetFileAsync(fullPath).Result.OpenAsync(FileAccess.Read).Result;
string key = System.Guid.NewGuid().ToString();
await tu.UploadAsync(fullPath, bucketName, key);
await VerifyUploadResponse(client, key);
await TestDownloadedFile(fullPath, tu, key);
}
}
catch (Exception e)
{
throw e;
}
});
}
public void SimpleUpload()
{
RunAsSync(async() =>
{
var fileExists = await this.BaseFolder.CheckExistsAsync(fullPath);
Console.WriteLine("File {0} exists = {1}", fullPath, fileExists);
using (var tu = new Amazon.S3.Transfer.TransferUtility(Client))
{
await tu.UploadAsync(fullPath, bucketName);
}
var response = await Client.GetObjectMetadataAsync(new GetObjectMetadataRequest
{
BucketName = bucketName,
Key = testFile
});
Console.WriteLine("S3 generated ETag: {0}", response.ETag);
Assert.IsTrue(response.ETag.Length > 0);
});
}
private async Task TestDownloadedFile(string fullPath, Amazon.S3.Transfer.TransferUtility tu, string key = null)
{
if (key == null)
{
key = testFile;
}
var downloadPath = fullPath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = key,
FilePath = downloadPath
};
var fileExists = await this.BaseFolder.CheckExistsAsync(downloadPath);
if (fileExists == ExistenceCheckResult.FileExists)
{
this.BaseFolder.GetFileAsync(downloadPath).Result.DeleteAsync().Wait();
}
fileExists = await this.BaseFolder.CheckExistsAsync(downloadPath);
Assert.IsTrue(fileExists == ExistenceCheckResult.NotFound);
await tu.DownloadAsync(downloadRequest);
await VerifyDownloadedFile(downloadPath);
// empty out file, except for 1 byte
var file = await this.BaseFolder.GetFileAsync(downloadPath);
await file.WriteAllTextAsync(testContent.Substring(0, 1));
await tu.DownloadAsync(downloadRequest);
await VerifyDownloadedFile(downloadPath);
}
public void SimpleUpload()
{
RunAsSync(async() =>
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
await tu.UploadAsync(fullPath, bucketName);
var response = await client.GetObjectMetadataAsync(new GetObjectMetadataRequest
{
BucketName = bucketName,
Key = testFile
});
Assert.IsTrue(response.ETag.Length > 0);
var downloadPath = fullPath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = testFile,
FilePath = downloadPath
};
var fileExists = await this.BaseFolder.CheckExistsAsync(downloadPath);
Assert.IsTrue(fileExists == ExistenceCheckResult.NotFound);
await tu.DownloadAsync(downloadRequest);
await TestDownloadedFile(downloadPath);
// empty out file, except for 1 byte
var file = await this.BaseFolder.GetFileAsync(downloadPath);
await file.WriteAllTextAsync(testContent.Substring(0, 1));
await tu.DownloadAsync(downloadRequest);
await TestDownloadedFile(downloadPath);
}
});
}
public string SerializeToS3(string secret)
{
using (MemoryStream stream = new MemoryStream()) {
// S3 filename
var key = "state_" + _jobstate.ToString();
// serialize to memory stream
IFormatter formatter = new BinaryFormatter();
formatter.Serialize(stream, this);
Console.Error.WriteLine("Upload stream size is: {0} bytes", stream.Length.ToString());
// upload Job to S3
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(_id, secret))
{
// set the stream position to the start of the stream
stream.Position = 0;
var tu = new Amazon.S3.Transfer.TransferUtility(client);
tu.Upload(stream, _s3bucket, key);
}
}
return(_jobstate.ToString());
}
public S3Uploader(string bucketName)
{
this.bucketName = bucketName;
this.transferUtility = new Amazon.S3.Transfer.TransferUtility("XXXXXXXXXXXXXXXXXX", "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
}
void ICoreAmazonS3.EndDownloadToFilePath(IAsyncResult result)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
transfer.EndDownload(result);
}
void ICoreAmazonS3.EndUploadObjectFromFilePath(IAsyncResult result)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
transfer.EndUpload(result);
}
void ICoreAmazonS3.UploadObjectFromStream(string bucketName, string objectKey, Stream stream, IDictionary<string, object> additionalProperties)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
InputStream = stream
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
transfer.Upload(request);
}
void ICoreAmazonS3.EndDownloadToFilePath(IAsyncResult result)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
transfer.EndDownload(result);
}
IAsyncResult ICoreAmazonS3.BeginDownloadToFilePath(string bucketName, string objectKey, string filepath, IDictionary<string, object> additionalProperties, AsyncCallback callback, object state)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return transfer.BeginDownload(request, callback, state);
}
void ICoreAmazonS3.EndUploadObjectFromFilePath(IAsyncResult result)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
transfer.EndUpload(result);
}
Task ICoreAmazonS3.DownloadToFilePathAsync(string bucketName, string objectKey, string filepath, IDictionary<string, object> additionalProperties, CancellationToken cancellationToken)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return transfer.DownloadAsync(request, cancellationToken);
}
IAsyncResult ICoreAmazonS3.BeginUploadObjectFromStream(string bucketName, string objectKey, Stream stream, IDictionary<string, object> additionalProperties, AsyncCallback callback, object state)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
InputStream = stream
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return transfer.BeginUpload(request, callback, state);
}
void ICoreAmazonS3.DownloadToFilePath(string bucketName, string objectKey, string filepath, IDictionary<string, object> additionalProperties)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = objectKey,
FilePath = filepath
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
transfer.Download(request);
}
static int Upload(string filepath, string bucket, string key)
{
try
{
var conf = ConfigurationManager.AppSettings;
using (var s3 = Amazon.AWSClientFactory.CreateAmazonS3Client(conf["AWSAccessKey"], conf["AWSSecretKey"]))
using (var tu = new Amazon.S3.Transfer.TransferUtility(s3))
tu.Upload(filepath, bucket, key);
return 0;
}
catch (FileNotFoundException)
{
Console.WriteLine(string.Format("File not found: {0}", filepath));
return 1;
}
catch (AmazonS3Exception e)
{
Console.WriteLine(string.Format("S3 Error: {0}", e.Message));
return 2;
}
catch (Exception e)
{
Console.WriteLine(string.Format("Unknown error: {0}", e));
return 3;
}
}
Task ICoreAmazonS3.UploadObjectFromStreamAsync(string bucketName, string objectKey, Stream stream, IDictionary<string, object> additionalProperties, CancellationToken cancellationToken)
{
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = objectKey,
InputStream = stream
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
return transfer.UploadAsync(request, cancellationToken);
}
public override Uri UploadWithoutQuota(string domain, string path, Stream stream, string contentType, string contentDisposition)
{
ACL acl = ACL.Auto;
using (AmazonS3 client = GetClient())
{
var util = new Amazon.S3.Transfer.TransferUtility(client);
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest();
string mime = string.IsNullOrEmpty(contentType)
? MimeMapping.GetMimeMapping(Path.GetFileName(path))
: contentType;
const int uploadTimeout = 10 * 60 * 1000;
request.BucketName = _bucket;
request.Key = MakePath(domain, path);
request.CannedACL = acl == ACL.Auto ? GetDomainACL(domain) : GetS3Acl(acl);
request.ContentType = mime;
request.Timeout = uploadTimeout;
var headers = new NameValueCollection();
headers.Add("Cache-Control", string.Format("public, maxage={0}", (int)TimeSpan.FromDays(5).TotalSeconds));
headers.Add("Etag", (DateTime.UtcNow.Ticks).ToString(CultureInfo.InvariantCulture));
headers.Add("Last-Modified", DateTime.UtcNow.ToString("R"));
headers.Add("Expires", DateTime.UtcNow.Add(TimeSpan.FromDays(5)).ToString("R"));
if (!string.IsNullOrEmpty(contentDisposition))
{
headers.Add("Content-Disposition", Uri.EscapeDataString(contentDisposition));
}
else if (mime == "application/octet-stream")
{
headers.Add("Content-Disposition", "attachment");
}
request.AddHeaders(headers);
//Send body
var buffered = stream.GetBuffered();
request.AutoCloseStream = false;
request.InputStream = buffered;
util.Upload(request);
InvalidateCloudFront(MakePath(domain, path));
return GetUri(domain, path);
}
}
Task ICoreAmazonS3.UploadObjectFromStorageAsync(string bucketName, string objectKey, IStorageFile storageFile, IDictionary<string, object> additionalProperties, CancellationToken cancellationToken)
{
var request = new Amazon.S3.Transfer.TransferUtilityUploadRequest();
request.BucketName = bucketName;
request.Key = objectKey;
request.StorageFile = storageFile;
InternalSDKUtils.ApplyValues(request, additionalProperties);
var transfer = new Amazon.S3.Transfer.TransferUtility(this);
return transfer.UploadAsync(request, cancellationToken);
}
