public ActionResult ViewStats(string id)
{
using (var client = Amazon.AWSClientFactory.CreateAmazonS3Client(AwsAccessKey, AwsSecretAccessKey))
{
var request = new GetObjectRequest();
request.WithBucketName("MyTwitterStats");
request.WithKey(id + ".json.gz");
GetObjectResponse response = null;
try
{
response = client.GetObject(request);
}
catch (AmazonS3Exception)
{
//TODO: Log exception.ErrorCode
return HttpNotFound();
}
using (response)
using (var stream = response.ResponseStream)
using (var zipStream = new GZipStream(stream, CompressionMode.Decompress))
using (var reader = new StreamReader(zipStream))
{
var jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings());
var stats = jsonSerializer.Deserialize<Stats>(new JsonTextReader(reader));
return View(stats);
}
}
}
public AmazonS3Object GetFile(string key)
{
var request = new GetObjectRequest();
request.WithBucketName(this._bucketName).WithKey(key);
try
{
var response = this._client.GetObject(request);
return response.ToAmazonS3Object();
}
catch (Exception ex)
{
Log.Error(string.Format("Cannot find the file with key {0}. Debug message: {1}", key, ex.Message));
return null;
}
}
public static byte[] GetFile(string name)
{
try
{
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(BucketName)
.WithKey(name);
AmazonS3Config config = new AmazonS3Config();
config.CommunicationProtocol = Protocol.HTTP;
using (MemoryStream ms = new MemoryStream())
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(AccessKeyID, SecretAccessKeyID, config))
using (GetObjectResponse response = client.GetObject(request))
{
if (response != null)
{
response.ResponseStream.CopyTo(ms);
return ms.ToArray();
}
else
return null;
}
}
catch (AmazonS3Exception amazonS3Exception)
{
#if(DEBUG)
if (amazonS3Exception.ErrorCode != null &&
(amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") ||
amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Console.WriteLine("Please check the provided AWS Credentials.");
Console.WriteLine("If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3");
}
else
{
Console.WriteLine("An error occurred with the message '{0}' when deleting an object", amazonS3Exception.Message);
}
#endif
return null;
}
catch (Exception e)
{
return null;
}
}
public static bool ReadLogFile(string s3FileName, string credentialFilePath)
{
Type t = System.Reflection.MethodBase.GetCurrentMethod().DeclaringType;
try
{
if (ReadS3Credentials(credentialFilePath) == false)
{
LogEvents.S3NoCredentials(t);
return false;
}
AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(_accessKeyId, _secretAccessKey);
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(_bucketName).WithKey(s3FileName);
S3Response responseWithMetadata = client.GetObject(request);
return true;
}
catch (AmazonS3Exception amazonS3Exception)
{
LogEvents.S3Error(t, amazonS3Exception);
return false;
}
}
/*
Sample call for upload:-
byte[] array = new byte[1024*1024*1024];
Random random = new Random();
random.NextBytes(array);
double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob");
double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob");
*
*
*/
public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType,
SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName=null, string blobName=null)
{
string accountName = ConfigurationManager.AppSettings.Get("AccountName");
string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
DateTime begin=DateTime.Now, end=DateTime.Now;
if (synchronizerType == SynchronizerType.Azure)
{
#region azure download/upload
if (containerName==null)
containerName = "testingraw";
if(blobName==null)
blobName = Guid.NewGuid().ToString();
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
if (syncDirection == SynchronizeDirection.Upload)
{
logger.Log("Start Stream Append");
container.CreateIfNotExist();
begin = DateTime.UtcNow;//////////////////////////////////////
try
{
using (MemoryStream memoryStream = new System.IO.MemoryStream(input))
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
blockBlob.UploadFromStream(memoryStream);
}
}
catch (Exception e)
{
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Append");
}
if (syncDirection == SynchronizeDirection.Download)
{
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
try
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
byte[] blobContents = blockBlob.DownloadByteArray();
//if (File.Exists(blobName))
// File.Delete(blobName);
begin = DateTime.UtcNow;//////////////////////////////////////
// using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate))
// {
byte[] contents = blockBlob.DownloadByteArray();
// fs.Write(contents, 0, contents.Length);
// }
}
catch (Exception e)
{
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
#endregion
}
else if (synchronizerType == SynchronizerType.AmazonS3)
{
#region amazon s3 stuff
if (containerName == null)
containerName = "testingraw";
if (blobName == null)
blobName = Guid.NewGuid().ToString();
AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey);
if (syncDirection == SynchronizeDirection.Upload)
{
ListBucketsResponse response = amazonS3Client.ListBuckets();
foreach (S3Bucket bucket in response.Buckets)
{
if (bucket.BucketName == containerName)
{
break;
}
}
amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName));
begin = DateTime.UtcNow;//////////////////////////////////////
MemoryStream ms = new MemoryStream();
ms.Write(input, 0, input.Length);
PutObjectRequest request = new PutObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
request.InputStream = ms;
amazonS3Client.PutObject(request);
end = DateTime.UtcNow;//////////////////////////////////////
}
if (syncDirection == SynchronizeDirection.Download)
{
if (File.Exists(blobName))
File.Delete(blobName);
begin = DateTime.UtcNow;//////////////////////////////////////
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
GetObjectResponse response = amazonS3Client.GetObject(request);
var localFileStream = File.Create(blobName);
response.ResponseStream.CopyTo(localFileStream);
localFileStream.Close();
end = DateTime.UtcNow;//////////////////////////////////////
}
#endregion
}
else
{
throw new InvalidDataException("syncronizer type is not valid");
}
return (end - begin).TotalMilliseconds;// return total time to upload in milliseconds
}
private void getAndSaveFile(string keyName, string storedFileName)
{
GetObjectRequest getRequest = new GetObjectRequest();
getRequest.WithBucketName("intelrecruiter")
.WithKey(keyName);
using (GetObjectResponse response = client.GetObject(getRequest))
{
string dest = Path.Combine(zipDirectoryName, storedFileName);
if (!File.Exists(dest))
{
response.WriteResponseStreamToFile(dest);
}
}
}
public byte[] UploadFileAsChunks(string filePath)
{
string s3objectName;
List<ChunkInfo> chunkList_cloud = new List<ChunkInfo>(); ; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
List<ChunkInfo> chunkList_local; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
try
{
if (logger != null) logger.Log("Start Synchronizer Check Blob Exists");
s3objectName = Path.GetFileName(filePath);
bool s3ObjectExists = S3ObjectExists(ChunkMetadataObjectPrefix + s3objectName);
if (logger != null) logger.Log("End Synchronizer Check Blob Exists");
if (s3ObjectExists)
{
if (logger != null) logger.Log("Start Synchronizer Fill Remote ChunkList");
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(ChunkMetadataObjectPrefix + s3objectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
StreamReader reader = new StreamReader(response.ResponseStream);
string chunkMD_JSON = reader.ReadToEnd();
FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON);
StaticChunkSize = fileMD.StaticChunkSize;
chunkList_cloud = fileMD.ChunkList;
if (logger != null) logger.Log("End Synchronizer Fill Remote ChunkList");
chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType);
chunkEncryptionType = SyncFactory.GetEncryptionType(fileMD.encryptionType);
}
if (logger != null) logger.Log("Start Synchronizer Fill Local ChunkList");
StaticChunk staticChunker = new StaticChunk(StaticChunkSize);
chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface
// structuredLog("I", "Number of chunks locally: " + chunkList_local.Count);
if (logger != null) logger.Log("End Synchronizer Fill Local ChunkList");
if (logger != null) logger.Log("Start Synchronizer ChunkList Compare");
List<ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud);
// structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count);
// structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count);
if (logger != null) logger.Log("End Synchronizer ChunkList Compare");
if (logger != null) logger.Log("Start Synchronizer Upload Multiple Chunks");
UploadChunkList(ref chunkList_toUpload, filePath, s3objectName);
if (logger != null) logger.Log("End Synchronizer Upload Multiple Chunks");
// structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count);
if (logger != null) logger.Log("Start Synchronizer ChunkList Upload");
string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter());
if (chunkList_toUpload.Count > 0) //upload new chunk list only if we uploaded some new chunks
{
UploadStringToS3Object(ChunkMetadataObjectPrefix + s3objectName, json);
}
SHA1 sha1 = new SHA1CryptoServiceProvider();
byte[] ret = sha1.ComputeHash(Encoding.ASCII.GetBytes(json));
if (logger != null) logger.Log("End Synchronizer ChunkList Upload");
return ret;
}
catch (Exception e)
{
structuredLog("E", " . UploadFileAsChunks: " + e);
return null;
}
}
public System.Tuple<List<ChunkInfo>, byte[]> GetObjectMetadata(string s3objectName)
{
List<ChunkInfo> retVal = null;
byte[] hash = null;
string metadataObjectName = ChunkMetadataObjectPrefix + s3objectName;
bool s3ObjectExists = S3ObjectExists(metadataObjectName);
if (s3ObjectExists)
{
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(metadataObjectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
StreamReader reader = new StreamReader(response.ResponseStream);
string chunkMD_JSON = reader.ReadToEnd();
FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON);
SHA1 sha1 = new SHA1CryptoServiceProvider();
hash = sha1.ComputeHash(Encoding.ASCII.GetBytes(chunkMD_JSON));
retVal = fileMD.ChunkList;
}
return new System.Tuple<List<ChunkInfo>, byte[]>(retVal, hash);
}
public bool DownloadS3ObjectToFile(string s3ObjectName, string filePath)
{
try
{
if (!S3ObjectExists(ChunkMetadataObjectPrefix + s3ObjectName))
return false;
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(s3ObjectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
if (File.Exists(filePath))
File.Delete(filePath);
var localFileStream = File.Create(filePath);
response.ResponseStream.CopyTo(localFileStream);
localFileStream.Close();
return true;
}
catch (Exception e)
{
structuredLog("E", "Exception in DownloadS3ObjectToFile: " + e);
return false;
}
}
public static void objectSerial(String filePath)
{
System.Console.WriteLine("\nhello,object!!");
NameValueCollection appConfig = ConfigurationManager.AppSettings;
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"]);
String bucketName = "chttest3";
String objectName = "hello";
//String filePath = "D:\\Visual Studio Project\\TestNetSDK\\TestNetSDK\\pic.jpg";
//PutBucket
System.Console.WriteLine("PutBucket: {0}\n",bucketName);
PutBucketResponse response = s3Client.PutBucket(new PutBucketRequest().WithBucketName(bucketName));
//PutObject
System.Console.WriteLine("PutObject!\n");
PutObjectRequest request = new PutObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(objectName);
request.WithFilePath(filePath);
PutObjectResponse PutResult = s3Client.PutObject(request);
System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult.ETag);
//HeadObject
System.Console.WriteLine("HeadObject!\n");
GetObjectMetadataResponse HeadResult = s3Client.GetObjectMetadata(new GetObjectMetadataRequest().WithBucketName(bucketName).WithKey(objectName));
System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult.ContentLength,HeadResult.ETag);
//GetObject
System.Console.WriteLine("GetObject!\n");
GetObjectResponse GetResult = s3Client.GetObject(new GetObjectRequest().WithBucketName(bucketName).WithKey(objectName).WithByteRange(1,15));
Stream responseStream = GetResult.ResponseStream;
StreamReader reader = new StreamReader(responseStream);
System.Console.WriteLine("Get Object Content:\n {0}\n", reader.ReadToEnd());
System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult.ETag);
//CopyObject
CopyObjectResponse CopyResult = s3Client.CopyObject(new CopyObjectRequest().WithSourceBucket(bucketName).WithSourceKey(objectName).WithDestinationBucket(bucketName).WithDestinationKey("hihi"));
System.Console.WriteLine("CopyObject: ETag: {0} \n",CopyResult.ETag);
//DeleteObject
System.Console.WriteLine("Delete Object!\n");
s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName));
s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi")); //copied object
//==============================jerry add ==============================
System.Console.WriteLine("Jerry Add!\n");
String objectName2 = "hello2";
NameValueCollection nvc = new NameValueCollection();
nvc.Add("A", "ABC");
System.Console.WriteLine("PutObject!\n");
PutObjectRequest request2 = new PutObjectRequest();
request2.WithBucketName(bucketName);
request2.WithKey(objectName2);
request2.WithAutoCloseStream(true);
request2.WithCannedACL(S3CannedACL.BucketOwnerFullControl);
request2.WithContentBody("test");
request2.WithContentType("test/xml");
request2.WithGenerateChecksum(true);
request2.WithMD5Digest("CY9rzUYh03PK3k6DJie09g==");
request2.WithMetaData(nvc);
request2.WithWebsiteRedirectLocation("http://hicloud.hinet.net/");
PutObjectResponse PutResult2 = s3Client.PutObject(request2);
System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult2.ETag);
System.Console.WriteLine("GetObject!\n");
GetObjectRequest request3 = new GetObjectRequest();
request3.WithBucketName(bucketName);
request3.WithKey(objectName2);
request3.WithByteRange(1, 2);
DateTime datetime = DateTime.UtcNow;
// 似乎不支援 datetime
request3.WithModifiedSinceDate(datetime.AddHours(-1));
request3.WithUnmodifiedSinceDate(datetime.AddHours(1));
request3.WithETagToMatch(PutResult2.ETag);
request3.WithETagToNotMatch("notMatch");
GetObjectResponse GetResult2 = s3Client.GetObject(request3);
Stream responseStream2 = GetResult2.ResponseStream;
StreamReader reader2 = new StreamReader(responseStream2);
System.Console.WriteLine("Get Object Content(es):\n {0}\n", reader2.ReadToEnd());
System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult2.ETag);
System.Console.WriteLine("HeadObject!\n");
GetObjectMetadataRequest request4 = new GetObjectMetadataRequest();
request4.WithBucketName(bucketName);
request4.WithKey(objectName2);
DateTime datetime2 = DateTime.UtcNow;
// 似乎不支援 datetime
request4.WithModifiedSinceDate(datetime2.AddHours(-1));
request4.WithUnmodifiedSinceDate(datetime2.AddHours(1));
request4.WithETagToMatch(PutResult2.ETag);
request4.WithETagToNotMatch("notMatch");
GetObjectMetadataResponse HeadResult2 = s3Client.GetObjectMetadata(request4);
System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult2.ContentLength, HeadResult2.ETag);
CopyObjectRequest request5 = new CopyObjectRequest();
request5.WithSourceBucket(bucketName);
request5.WithSourceKey(objectName2);
request5.WithDestinationBucket(bucketName);
request5.WithDestinationKey("hihi2");
DateTime datetime3 = DateTime.UtcNow;
// 似乎不支援 datetime
request5.WithModifiedSinceDate(datetime3.AddHours(-1));
request5.WithUnmodifiedSinceDate(datetime3.AddHours(1));
request5.WithETagToMatch(PutResult2.ETag);
request5.WithETagToNotMatch("notMatch");
request5.WithMetaData(nvc);
request5.WithCannedACL(S3CannedACL.PublicRead);
request5.WithContentType("test/xml");
request5.WithWebsiteRedirectLocation("http://hicloud.hinet.net/");
CopyObjectResponse CopyResult2 = s3Client.CopyObject(request5);
System.Console.WriteLine("CopyObject: ETag: {0} \n", CopyResult2.ETag);
System.Console.WriteLine("Delete Object!\n");
s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName2));
s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi2")); //copied object
//==============================jerry add end==============================
//DeleteBucket
System.Console.WriteLine("Delete Bucket!\n");
//s3Client.DeletesBucket(new DeleteBucketRequest().WithBucketName(bucketName));
s3Client.DeleteBucket(new DeleteBucketRequest().WithBucketName(bucketName));
System.Console.WriteLine("END!");
}
private void Download(string localFolderName, ProcessItem file)
{
using (var client = CreateAmazonS3Client()) {
file.Action = ProcessAction.DownloadingException;
if (client == null) return;
try {
var request = new GetObjectRequest();
request.WithBucketName(_bucketName).WithKey(file.S3Path);
using (var response = client.GetObject(request)) {
var fileName = "";
if (response.Metadata != null && response.Metadata.Count > 0) {
var localPath = response.Metadata.Get("x-amz-meta-localpath");
if (String.IsNullOrWhiteSpace(localPath)) return;
localPath = HttpUtility.UrlDecode(localPath);
if (localPath != null) {
var localPathArr = localPath.Split('\\');
var rootDirectory = localFolderName;
for (var i = 1; i < localPathArr.Length - 1; i++) {
rootDirectory = Path.Combine(rootDirectory, localPathArr[i]);
if(!Directory.Exists(rootDirectory))
Directory.CreateDirectory(rootDirectory);
}
fileName = Path.Combine(rootDirectory, localPathArr[localPathArr.Length - 1]);
}
}
if (String.IsNullOrWhiteSpace(fileName)) return;
using (var fileStream = new FileStream(fileName, FileMode.Create)) {
using (var stream = response.ResponseStream) {
var data = new byte[32768];
int bytesRead;
do {
bytesRead = stream.Read(data, 0, data.Length);
fileStream.Write(data, 0, bytesRead);
} while (bytesRead > 0);
fileStream.Flush();
file.Action = ProcessAction.DownloadingDone;
}
}
}
} catch (Exception) {
}
}
}
private void getLastSession()
{
string cookieName;
string cookieValue;
using (client = Amazon.AWSClientFactory.CreateAmazonS3Client("AKIAJ47VSG7WMA62WLCA", "3tqlHujlftpk6j/z5OtDw2eg9N2FJtz1RwL8bEa3"))
{
GetObjectRequest txtRequest = new GetObjectRequest();
txtRequest.WithBucketName("intelrecruiter")
.WithKey(nsbeCookieValueKey);
var response = client.GetObject(txtRequest);
StreamReader reader = new StreamReader(response.ResponseStream);
cookieValue = reader.ReadToEnd();
response.Dispose();
txtRequest = new GetObjectRequest();
txtRequest.WithBucketName("intelrecruiter")
.WithKey(nsbeCookieNameKey);
response = client.GetObject(txtRequest);
reader = new StreamReader(response.ResponseStream);
cookieName = reader.ReadToEnd();
response.Dispose();
}
nsbeCookie = new Cookie(cookieName, cookieValue, "/");
}
public string GetObjectInformation(string bucketName, string objectKey)
{
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName).WithKey(objectKey);
GetObjectResponse response = _client.GetObject(request);
return response.AmazonId2;
}
public byte[] DownloadS3ObjectToBytes(string s3ObjectName)
{
try
{
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(s3ObjectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
byte[] buffer = new byte[1024];
using (MemoryStream ms = new MemoryStream())
{
int read;
while ((read = response.ResponseStream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
return ms.ToArray();
}
}
catch (Exception e)
{
structuredLog("E", "Exception in DownloadS3ObjectToBytes: " + e);
return null;
}
}