/// <summary>
/// Gets the file stream.
/// </summary>
/// <param name="fileUrl"></param>
/// <returns></returns>
public Stream GetFileStream(string fileUrl)
{
_logger.LogTrace("Entering GetFileStream in AmazonS3UploadService");
try
{
if (!string.IsNullOrWhiteSpace(fileUrl))
{
string fileNameInS3 = GetAwsS3FileName(ref fileUrl);
// Getting bucket name from fileUrl
var bucketName = GetAwsS3BucketName(fileUrl, fileNameInS3);
var regionEndPoint = RegionEndpoint.GetBySystemName(_awsParams.Region);
// Initializing AmazonS3Client with AWS credentials
IAmazonS3 s3Client = new AmazonS3Client(_awsParams.AccessKeyId, _awsParams.SecretKey, regionEndPoint);
var utility = new TransferUtility(s3Client);
return(utility.OpenStream(bucketName, fileNameInS3));
}
return(null);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
throw new InvalidOperationException("GetFileStream failed in AmazonS3UploadService");
}
}
private IEnumerable <DrugExposure> GetDrugExposures(string key)
{
key = key.Replace("PERSON", "DRUG_EXPOSURE");
if (GetKeys(key).Any())
{
using (var transferUtility = new TransferUtility(_awsAccessKeyId, _awsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var responseStream = transferUtility.OpenStream(_bucket, key))
using (var bufferedStream = new BufferedStream(responseStream))
using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress))
using (var reader = new StreamReader(gzipStream, Encoding.Default))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!string.IsNullOrEmpty(line))
{
var row = line.Split('\t');
var personId = long.Parse(row[1]);
var conceptId = int.Parse(row[2]);
var start = DateTime.Parse(row[3]);
yield return(new DrugExposure(new Entity())
{
PersonId = personId,
StartDate = start,
ConceptId = conceptId
});
}
}
}
}
}
public S3DataReader2(string bucket, string folder, string awsAccessKeyId,
string awsSecretAccessKey, int chunkId, string fileName, Dictionary <string, int> fieldHeaders, string prefix, long initRow)
{
_chunkId = chunkId;
_bucket = bucket;
_folder = folder;
_fileName = fileName;
_fieldHeaders = fieldHeaders;
_prefix = prefix;
_spliter = new StringSplitter(this._fieldHeaders.Count);
_transferUtility = new TransferUtility(awsAccessKeyId, awsSecretAccessKey, Amazon.RegionEndpoint.USEast1);
_responseStream = _transferUtility.OpenStream(bucket, $"{folder}/{chunkId}/{fileName}/{fileName}{prefix}_part_00.gz");
_bufferedStream = new BufferedStream(_responseStream);
_gzipStream = new GZipStream(_bufferedStream, CompressionMode.Decompress);
_reader = new StreamReader(_gzipStream, Encoding.Default);
_rowIndex = initRow;
for (var i = 0; i < initRow; i++)
{
_reader.ReadLine();
}
if (initRow > 0)
{
Console.WriteLine($"{fileName}{prefix}_part_00.gz; Rows skipped={initRow}");
}
}
private IEnumerable <ObservationPeriod> GetObservationPeriods(string key)
{
key = key.Replace("PERSON", "OBSERVATION_PERIOD");
//if (GetKeys(key).Any())
{
using (var transferUtility = new TransferUtility(_awsAccessKeyId, _awsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var responseStream = transferUtility.OpenStream(_bucket, key))
using (var bufferedStream = new BufferedStream(responseStream))
using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress))
using (var reader = new StreamReader(gzipStream, Encoding.Default))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!string.IsNullOrEmpty(line))
{
var row = line.Split('\t');
var personId = long.Parse(row[1]);
var start = DateTime.Parse(row[2]);
var end = DateTime.Parse(row[3]);
yield return(new ObservationPeriod {
PersonId = personId, StartDate = start, EndDate = end
});
}
}
}
}
}
private IEnumerable <Person> GetPersons(string key)
{
using (var transferUtility = new TransferUtility(_awsAccessKeyId, _awsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var responseStream = transferUtility.OpenStream(_bucket, key))
using (var bufferedStream = new BufferedStream(responseStream))
using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress))
using (var reader = new StreamReader(gzipStream, Encoding.Default))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!string.IsNullOrEmpty(line))
{
var row = line.Split('\t');
var personId = long.Parse(row[0]);
var genderConceptId = int.Parse(row[1]);
var yearOfBirth = int.Parse(row[2]);
yield return(new Person(new Entity())
{
PersonId = personId, GenderConceptId = genderConceptId, YearOfBirth = yearOfBirth
});
}
}
}
}
public async Task <IActionResult> Download(int id)
{
if (id == 0)
{
return(Content("File not found"));
}
var userId = userManager.GetUserId(HttpContext.User);
var result = dbContext.OcrElements.Where(x => x.UserId == userId && x.Id == id).FirstOrDefault();
if (result == null)
{
return(Content("File not found"));
}
var memory = new MemoryStream();
var downloadRequest = new TransferUtilityOpenStreamRequest
{
Key = result.ImageFilenamePath,
BucketName = _awsAccess.S3BucketName,
};
using (var stream = fileTransferUtility.OpenStream(downloadRequest))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
return(File(memory, result.ImageFileContentType, result.ImageFilename));
}
public Stream getMyFilesFromS3(string bucketName, string subDirectoryInBucket, string fileNameInS3)
{
string accessKey = "AKIAWXS64BGGY47QHRWQ";
string secretKey = "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";
AmazonS3Client client = new AmazonS3Client(accessKey, secretKey, RegionEndpoint.USEast2);
// input explained :
// localFilePath = the full local file path e.g. "c:\mydir\mysubdir\myfilename.zip"
// bucketName : the name of the bucket in S3 ,the bucket should be alreadt created
// subDirectoryInBucket : if this string is not empty the file will be uploaded to
// a subdirectory with this name
// fileNameInS3 = the file name in the S3
// create an instance of IAmazonS3 class ,in my case i choose RegionEndpoint.EUWest1
// you can change that to APNortheast1 , APSoutheast1 , APSoutheast2 , CNNorth1
// SAEast1 , USEast1 , USGovCloudWest1 , USWest1 , USWest2 . this choice will not
// store your file in a different cloud storage but (i think) it differ in performance
// depending on your location
//IAmazonS3 client = new AmazonS3Client(RegionEndpoint.USEast2);
// create a TransferUtility instance passing it the IAmazonS3 created in the first step
TransferUtility utility = new TransferUtility(client);
// making a TransferUtilityUploadRequest instance
TransferUtilityOpenStreamRequest request = new TransferUtilityOpenStreamRequest();
if (subDirectoryInBucket == "" || subDirectoryInBucket == null)
{
request.BucketName = bucketName; //no subdirectory just bucket name
}
else
{ // subdirectory and bucket name
request.BucketName = bucketName + @"/" + subDirectoryInBucket;
}
request.Key = fileNameInS3; //file name up in S3
Stream stream = utility.OpenStream(request); //commensing the transfer
return(stream); //indicate that the file was sent
}
public void ServerSideEncryptionBYOKTransferUtility()
{
var bucketName = S3TestUtils.CreateBucket(Client);
try
{
Aes aesEncryption = Aes.Create();
aesEncryption.KeySize = 256;
aesEncryption.GenerateKey();
string base64Key = Convert.ToBase64String(aesEncryption.Key);
TransferUtility utility = new TransferUtility(Client);
var uploadRequest = new TransferUtilityUploadRequest
{
BucketName = bucketName,
Key = key,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
uploadRequest.InputStream = new MemoryStream(UTF8Encoding.UTF8.GetBytes("Encrypted Content"));
utility.Upload(uploadRequest);
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest
{
BucketName = bucketName,
Key = key,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
GetObjectMetadataResponse getObjectMetadataResponse = Client.GetObjectMetadata(getObjectMetadataRequest);
Assert.AreEqual(ServerSideEncryptionCustomerMethod.AES256, getObjectMetadataResponse.ServerSideEncryptionCustomerMethod);
var openRequest = new TransferUtilityOpenStreamRequest
{
BucketName = bucketName,
Key = key,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
using (var stream = new StreamReader(utility.OpenStream(openRequest)))
{
var content = stream.ReadToEnd();
Assert.AreEqual(content, "Encrypted Content");
}
}
finally
{
AmazonS3Util.DeleteS3BucketWithObjects(Client, bucketName);
}
}
//public Stream GetResourceStreamPartial(string resourceID, long start, long? end)
//{
// MemoryStream memStram = new MemoryStream();
// IAmazonS3 s3Client = new AmazonS3Client();
// GetObjectRequest request = new GetObjectRequest
// {
// BucketName = "piccoli",
// Key = resourceID
// };
// if(end != null)
// {
// request.ByteRange = new ByteRange(start, (long)end);
// }
// using (GetObjectResponse response = s3Client.GetObject(request))
// {
// //string dest = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), keyName);
// //if (!File.Exists(dest))
// {
// Stream returnStream = response.ResponseStream;
// returnStream.CopyTo(memStram);
// }
// }
// return memStram;
//}
public Stream GetResourceStream(string resourceID, long start, long end)
{
IAmazonS3 s3Client = new AmazonS3Client();
using (TransferUtility tr = new TransferUtility(s3Client))
{
return(tr.OpenStream("piccoli", resourceID));
}
}
private Stream Download(string key)
{
var stream = _transferUtility.OpenStream(new TransferUtilityOpenStreamRequest()
{
BucketName = _amazonS3StorageConfiguration.AWSFileBucket,
Key = key,
});
return(stream);
}
public void Update(string id, byte[] fileBytes)
{
using var client = new AmazonS3Client(AccessId, AccessKey, Region);
var fileTransferUtility = new TransferUtility(client);
using var stream = fileTransferUtility.OpenStream(BucketName, id);
stream.Write(fileBytes);
}
public Stream GetResourceStream(string resourceID)
{
IAmazonS3 s3Client = new AmazonS3Client();
Stream ms = new MemoryStream();
using (TransferUtility tr = new TransferUtility(s3Client))
{
Stream amazonStream = tr.OpenStream("piccoli", resourceID);
ms = Amazon.S3.Util.AmazonS3Util.MakeStreamSeekable(amazonStream);
}
return(ms);
}
public byte[] Select(string id)
{
using var client = new AmazonS3Client(AccessId, AccessKey, Region);
var fileTransferUtility = new TransferUtility(client);
using var stream = fileTransferUtility.OpenStream(BucketName, id);
byte[] fileBytes = new byte[stream.Length];
stream.Read(fileBytes, 0, fileBytes.Length);
return(fileBytes);
}
/// <summary>
/// Opens a stream of the content from Amazon S3
/// </summary>
/// <param name="key">The key under which the Amazon S3 object is stored.</param>
/// <param name="version">The identifier for the specific version of the object to be downloaded, if required.</param>
/// <param name="settings">The <see cref="DownloadSettings"/> required to download from Amazon S3.</param>
/// <returns>A stream.</returns>
public Stream Open(string key, string version, DownloadSettings settings)
{
TransferUtility utility = this.GetUtility(settings);
TransferUtilityOpenStreamRequest request = this.CreateOpenRequest(settings);
request.Key = key;
if (!String.IsNullOrEmpty(version))
{
request.VersionId = version;
}
_Log.Verbose("Opening stream {0} from bucket {1}...", key, settings.BucketName);
return(utility.OpenStream(request));
}
private IEnumerable <Measurement> GetMeasurements(string key)
{
key = key.Replace("PERSON", "MEASUREMENT");
if (GetKeys(key).Any())
{
using (var transferUtility = new TransferUtility(_awsAccessKeyId, _awsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var responseStream = transferUtility.OpenStream(_bucket, key))
using (var bufferedStream = new BufferedStream(responseStream))
using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress))
using (var reader = new StreamReader(gzipStream, Encoding.Default))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!string.IsNullOrEmpty(line))
{
var row = line.Split('\t');
var personId = long.Parse(row[1]);
var conceptId = int.Parse(row[2]);
var start = DateTime.Parse(row[3]);
decimal?valueAsNumber = null;
if (decimal.TryParse(row[8], out var num))
{
valueAsNumber = num;
}
var valueSourceValue = row[19];
if (valueSourceValue == "\0")
{
valueSourceValue = null;
}
yield return(new Measurement(new Entity())
{
PersonId = personId,
StartDate = start,
ConceptId = conceptId,
ValueAsNumber = valueAsNumber,
ValueSourceValue = valueSourceValue
});
}
}
}
}
}
public byte[] DownloadFileByteContents(string remoteFilename)
{
byte[] buffer = null;
TransferUtilityOpenStreamRequest req = new TransferUtilityOpenStreamRequest();
req.BucketName = CurrentBucketName;
req.Key = remoteFilename;
using (Stream s = _transfer.OpenStream(req))
{
buffer = new byte[s.Length];
s.Position = 0;
s.Read(buffer, 0, (int)s.Length);
s.Flush();
}
return(buffer);
}
public void BlockTransferUtilityOpenStreamTest()
{
var transferUtility = new TransferUtility(RegionEndpoint.USWest2);
try
{
transferUtility.OpenStream(new TransferUtilityOpenStreamRequest
{
BucketName = "arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint/mybanner",
Key = "test.txt"
});
}
catch (AmazonS3Exception e)
{
var expectedMessage = "OpenStream does not support S3 Object Lambda resources";
Assert.AreEqual(expectedMessage, e.Message);
}
}
private bool GetRestorePoint(Amazon.S3.Util.S3EventNotification.S3Entity s3Event)
{
var attempt = 0;
while (true)
{
try
{
attempt++;
var msg = new StringBuilder();
var timer = new Stopwatch();
timer.Start();
using (var transferUtility = new TransferUtility(this.S3Client))
using (var responseStream = transferUtility.OpenStream(s3Event.Bucket.Name, s3Event.Object.Key))
using (var reader = new StreamReader(responseStream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
var fileName = line.Split(':')[0];
var rowIndex = long.Parse(line.Split(':')[1]);
_restorePoint.Add(fileName, rowIndex);
msg.Append($"{fileName}:{rowIndex};");
}
}
timer.Stop();
Console.WriteLine("Restore point:" + msg + " | " + timer.ElapsedMilliseconds + "ms");
return(true);
}
catch (Exception e)
{
if (attempt > 5)
{
Console.WriteLine($"WARN_EXC - GetRestorePoint [{s3Event.Object.Key}]");
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
throw;
}
}
}
}
public async Task OpenStreamTest()
{
var fileName = UtilityMethods.GenerateName(Path.Combine("OpenStreamTest", "File"));
var key = fileName;
var originalFilePath = Path.Combine(basePath, fileName);
UtilityMethods.GenerateFile(originalFilePath, 2 * MEG_SIZE);
await Client.PutObjectAsync(new PutObjectRequest
{
BucketName = testBucketName,
Key = key,
FilePath = originalFilePath
}).ConfigureAwait(false);
using (var transferUtility = new TransferUtility(Client))
using (var stream = transferUtility.OpenStream(testBucketName, key))
{
Assert.NotNull(stream);
Assert.True(stream.CanRead);
}
}
public void OpenStreamTest()
{
var fileName = UtilityMethods.GenerateName(@"OpenStreamTest\File");
var key = fileName;
var originalFilePath = Path.Combine(basePath, fileName);
UtilityMethods.GenerateFile(originalFilePath, 2 * MEG_SIZE);
Client.PutObject(new PutObjectRequest
{
BucketName = bucketName,
Key = key,
FilePath = originalFilePath
});
var transferUtility = new TransferUtility(Client);
var stream = transferUtility.OpenStream(bucketName, key);
Assert.IsNotNull(stream);
Assert.IsTrue(stream.CanRead);
stream.Close();
}
private byte[] DownloadFileFromMuiDir(string fileName)
{
using (IAmazonS3 s3Client = CreateS3Client())
{
var fileKey = $"{_secretsDirectory}/{fileName}";
var s3FileInfo = new S3FileInfo(s3Client, _bucketName, fileKey);
if (!s3FileInfo.Exists)
{
return(null);
}
using (var fileTransferUtility = new TransferUtility(s3Client))
using (var stream = fileTransferUtility.OpenStream(_bucketName, fileKey))
using (var ms = new MemoryStream((int)s3FileInfo.Length))
{
stream.CopyTo(ms);
var bytes = ms.ToArray();
return(bytes);
}
}
}
public static byte[] s3get(string filename, bool removeOnFetch = false)
{
log("Retrieving s3://{0}", filename);
try
{
using (IAmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(@"AKIAJ2UM2AGXBMIPF5EQ", @"UKtB97UANKXPhRI3edldqMcDAEAx7sgtsBImDK1B", RegionEndpoint.USWest2))
{
TransferUtility tu = new TransferUtility(client);
using (MemoryStream ms = new MemoryStream())
using (Stream res = tu.OpenStream(@"osu-central", filename))
{
byte[] buffer = new byte[65536];
int bytesRead = -1;
while ((bytesRead = res.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, bytesRead);
}
if (removeOnFetch)
{
client.DeleteObject(new DeleteObjectRequest()
{
BucketName = @"osu-central", Key = filename
});
}
return(ms.ToArray());
}
}
}
catch (Exception e)
{
log(e);
return(null);
}
}
public MatrixCalculation GetCalculation(string id)
{
var c = serializer.Deserialize <MatrixCalculation>(transferUtility.OpenStream(bucketName, id));
return(c);
}
