public ActionResult ViewStats(string id) { using (var client = Amazon.AWSClientFactory.CreateAmazonS3Client(AwsAccessKey, AwsSecretAccessKey)) { var request = new GetObjectRequest(); request.WithBucketName("MyTwitterStats"); request.WithKey(id + ".json.gz"); GetObjectResponse response = null; try { response = client.GetObject(request); } catch (AmazonS3Exception) { //TODO: Log exception.ErrorCode return HttpNotFound(); } using (response) using (var stream = response.ResponseStream) using (var zipStream = new GZipStream(stream, CompressionMode.Decompress)) using (var reader = new StreamReader(zipStream)) { var jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings()); var stats = jsonSerializer.Deserialize<Stats>(new JsonTextReader(reader)); return View(stats); } } }
/* Sample call for upload:- byte[] array = new byte[1024*1024*1024]; Random random = new Random(); random.NextBytes(array); double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob"); double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob"); * * */ public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType, SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName=null, string blobName=null) { string accountName = ConfigurationManager.AppSettings.Get("AccountName"); string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); DateTime begin=DateTime.Now, end=DateTime.Now; if (synchronizerType == SynchronizerType.Azure) { #region azure download/upload if (containerName==null) containerName = "testingraw"; if(blobName==null) blobName = Guid.NewGuid().ToString(); CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); if (syncDirection == SynchronizeDirection.Upload) { logger.Log("Start Stream Append"); container.CreateIfNotExist(); begin = DateTime.UtcNow;////////////////////////////////////// try { using (MemoryStream memoryStream = new System.IO.MemoryStream(input)) { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); blockBlob.UploadFromStream(memoryStream); } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Append"); } if (syncDirection == SynchronizeDirection.Download) { logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); try { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); byte[] blobContents = blockBlob.DownloadByteArray(); //if (File.Exists(blobName)) // File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// // using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate)) // { byte[] contents = blockBlob.DownloadByteArray(); // fs.Write(contents, 0, contents.Length); // } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } #endregion } else if (synchronizerType == SynchronizerType.AmazonS3) { #region amazon s3 stuff if (containerName == null) containerName = "testingraw"; if (blobName == null) blobName = Guid.NewGuid().ToString(); AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey); if (syncDirection == SynchronizeDirection.Upload) { ListBucketsResponse response = amazonS3Client.ListBuckets(); foreach (S3Bucket bucket in response.Buckets) { if (bucket.BucketName == containerName) { break; } } amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName)); begin = DateTime.UtcNow;////////////////////////////////////// MemoryStream ms = new MemoryStream(); ms.Write(input, 0, input.Length); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); request.InputStream = ms; amazonS3Client.PutObject(request); end = DateTime.UtcNow;////////////////////////////////////// } if (syncDirection == SynchronizeDirection.Download) { if (File.Exists(blobName)) File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); GetObjectResponse response = amazonS3Client.GetObject(request); var localFileStream = File.Create(blobName); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); end = DateTime.UtcNow;////////////////////////////////////// } #endregion } else { throw new InvalidDataException("syncronizer type is not valid"); } return (end - begin).TotalMilliseconds;// return total time to upload in milliseconds }
public byte[] UploadFileAsChunks(string filePath) { string s3objectName; List<ChunkInfo> chunkList_cloud = new List<ChunkInfo>(); ; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....) List<ChunkInfo> chunkList_local; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....) try { if (logger != null) logger.Log("Start Synchronizer Check Blob Exists"); s3objectName = Path.GetFileName(filePath); bool s3ObjectExists = S3ObjectExists(ChunkMetadataObjectPrefix + s3objectName); if (logger != null) logger.Log("End Synchronizer Check Blob Exists"); if (s3ObjectExists) { if (logger != null) logger.Log("Start Synchronizer Fill Remote ChunkList"); GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(ChunkMetadataObjectPrefix + s3objectName); GetObjectResponse response = amazonS3Client.GetObject(request); StreamReader reader = new StreamReader(response.ResponseStream); string chunkMD_JSON = reader.ReadToEnd(); FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON); StaticChunkSize = fileMD.StaticChunkSize; chunkList_cloud = fileMD.ChunkList; if (logger != null) logger.Log("End Synchronizer Fill Remote ChunkList"); chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType); chunkEncryptionType = SyncFactory.GetEncryptionType(fileMD.encryptionType); } if (logger != null) logger.Log("Start Synchronizer Fill Local ChunkList"); StaticChunk staticChunker = new StaticChunk(StaticChunkSize); chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface // structuredLog("I", "Number of chunks locally: " + chunkList_local.Count); if (logger != null) logger.Log("End Synchronizer Fill Local ChunkList"); if (logger != null) logger.Log("Start Synchronizer ChunkList Compare"); List<ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud); // structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count); // structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count); if (logger != null) logger.Log("End Synchronizer ChunkList Compare"); if (logger != null) logger.Log("Start Synchronizer Upload Multiple Chunks"); UploadChunkList(ref chunkList_toUpload, filePath, s3objectName); if (logger != null) logger.Log("End Synchronizer Upload Multiple Chunks"); // structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count); if (logger != null) logger.Log("Start Synchronizer ChunkList Upload"); string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter()); if (chunkList_toUpload.Count > 0) //upload new chunk list only if we uploaded some new chunks { UploadStringToS3Object(ChunkMetadataObjectPrefix + s3objectName, json); } SHA1 sha1 = new SHA1CryptoServiceProvider(); byte[] ret = sha1.ComputeHash(Encoding.ASCII.GetBytes(json)); if (logger != null) logger.Log("End Synchronizer ChunkList Upload"); return ret; } catch (Exception e) { structuredLog("E", " . UploadFileAsChunks: " + e); return null; } }
public System.Tuple<List<ChunkInfo>, byte[]> GetObjectMetadata(string s3objectName) { List<ChunkInfo> retVal = null; byte[] hash = null; string metadataObjectName = ChunkMetadataObjectPrefix + s3objectName; bool s3ObjectExists = S3ObjectExists(metadataObjectName); if (s3ObjectExists) { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(metadataObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); StreamReader reader = new StreamReader(response.ResponseStream); string chunkMD_JSON = reader.ReadToEnd(); FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON); SHA1 sha1 = new SHA1CryptoServiceProvider(); hash = sha1.ComputeHash(Encoding.ASCII.GetBytes(chunkMD_JSON)); retVal = fileMD.ChunkList; } return new System.Tuple<List<ChunkInfo>, byte[]>(retVal, hash); }
public bool DownloadS3ObjectToFile(string s3ObjectName, string filePath) { try { if (!S3ObjectExists(ChunkMetadataObjectPrefix + s3ObjectName)) return false; GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(s3ObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); if (File.Exists(filePath)) File.Delete(filePath); var localFileStream = File.Create(filePath); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); return true; } catch (Exception e) { structuredLog("E", "Exception in DownloadS3ObjectToFile: " + e); return false; } }
public static void objectSerial(String filePath) { System.Console.WriteLine("\nhello,object!!"); NameValueCollection appConfig = ConfigurationManager.AppSettings; AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"]); String bucketName = "chttest3"; String objectName = "hello"; //String filePath = "D:\\Visual Studio Project\\TestNetSDK\\TestNetSDK\\pic.jpg"; //PutBucket System.Console.WriteLine("PutBucket: {0}\n",bucketName); PutBucketResponse response = s3Client.PutBucket(new PutBucketRequest().WithBucketName(bucketName)); //PutObject System.Console.WriteLine("PutObject!\n"); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(bucketName); request.WithKey(objectName); request.WithFilePath(filePath); PutObjectResponse PutResult = s3Client.PutObject(request); System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult.ETag); //HeadObject System.Console.WriteLine("HeadObject!\n"); GetObjectMetadataResponse HeadResult = s3Client.GetObjectMetadata(new GetObjectMetadataRequest().WithBucketName(bucketName).WithKey(objectName)); System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult.ContentLength,HeadResult.ETag); //GetObject System.Console.WriteLine("GetObject!\n"); GetObjectResponse GetResult = s3Client.GetObject(new GetObjectRequest().WithBucketName(bucketName).WithKey(objectName).WithByteRange(1,15)); Stream responseStream = GetResult.ResponseStream; StreamReader reader = new StreamReader(responseStream); System.Console.WriteLine("Get Object Content:\n {0}\n", reader.ReadToEnd()); System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult.ETag); //CopyObject CopyObjectResponse CopyResult = s3Client.CopyObject(new CopyObjectRequest().WithSourceBucket(bucketName).WithSourceKey(objectName).WithDestinationBucket(bucketName).WithDestinationKey("hihi")); System.Console.WriteLine("CopyObject: ETag: {0} \n",CopyResult.ETag); //DeleteObject System.Console.WriteLine("Delete Object!\n"); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName)); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi")); //copied object //==============================jerry add ============================== System.Console.WriteLine("Jerry Add!\n"); String objectName2 = "hello2"; NameValueCollection nvc = new NameValueCollection(); nvc.Add("A", "ABC"); System.Console.WriteLine("PutObject!\n"); PutObjectRequest request2 = new PutObjectRequest(); request2.WithBucketName(bucketName); request2.WithKey(objectName2); request2.WithAutoCloseStream(true); request2.WithCannedACL(S3CannedACL.BucketOwnerFullControl); request2.WithContentBody("test"); request2.WithContentType("test/xml"); request2.WithGenerateChecksum(true); request2.WithMD5Digest("CY9rzUYh03PK3k6DJie09g=="); request2.WithMetaData(nvc); request2.WithWebsiteRedirectLocation("http://hicloud.hinet.net/"); PutObjectResponse PutResult2 = s3Client.PutObject(request2); System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult2.ETag); System.Console.WriteLine("GetObject!\n"); GetObjectRequest request3 = new GetObjectRequest(); request3.WithBucketName(bucketName); request3.WithKey(objectName2); request3.WithByteRange(1, 2); DateTime datetime = DateTime.UtcNow; // 似乎不支援 datetime request3.WithModifiedSinceDate(datetime.AddHours(-1)); request3.WithUnmodifiedSinceDate(datetime.AddHours(1)); request3.WithETagToMatch(PutResult2.ETag); request3.WithETagToNotMatch("notMatch"); GetObjectResponse GetResult2 = s3Client.GetObject(request3); Stream responseStream2 = GetResult2.ResponseStream; StreamReader reader2 = new StreamReader(responseStream2); System.Console.WriteLine("Get Object Content(es):\n {0}\n", reader2.ReadToEnd()); System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult2.ETag); System.Console.WriteLine("HeadObject!\n"); GetObjectMetadataRequest request4 = new GetObjectMetadataRequest(); request4.WithBucketName(bucketName); request4.WithKey(objectName2); DateTime datetime2 = DateTime.UtcNow; // 似乎不支援 datetime request4.WithModifiedSinceDate(datetime2.AddHours(-1)); request4.WithUnmodifiedSinceDate(datetime2.AddHours(1)); request4.WithETagToMatch(PutResult2.ETag); request4.WithETagToNotMatch("notMatch"); GetObjectMetadataResponse HeadResult2 = s3Client.GetObjectMetadata(request4); System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult2.ContentLength, HeadResult2.ETag); CopyObjectRequest request5 = new CopyObjectRequest(); request5.WithSourceBucket(bucketName); request5.WithSourceKey(objectName2); request5.WithDestinationBucket(bucketName); request5.WithDestinationKey("hihi2"); DateTime datetime3 = DateTime.UtcNow; // 似乎不支援 datetime request5.WithModifiedSinceDate(datetime3.AddHours(-1)); request5.WithUnmodifiedSinceDate(datetime3.AddHours(1)); request5.WithETagToMatch(PutResult2.ETag); request5.WithETagToNotMatch("notMatch"); request5.WithMetaData(nvc); request5.WithCannedACL(S3CannedACL.PublicRead); request5.WithContentType("test/xml"); request5.WithWebsiteRedirectLocation("http://hicloud.hinet.net/"); CopyObjectResponse CopyResult2 = s3Client.CopyObject(request5); System.Console.WriteLine("CopyObject: ETag: {0} \n", CopyResult2.ETag); System.Console.WriteLine("Delete Object!\n"); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName2)); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi2")); //copied object //==============================jerry add end============================== //DeleteBucket System.Console.WriteLine("Delete Bucket!\n"); //s3Client.DeletesBucket(new DeleteBucketRequest().WithBucketName(bucketName)); s3Client.DeleteBucket(new DeleteBucketRequest().WithBucketName(bucketName)); System.Console.WriteLine("END!"); }
public byte[] DownloadS3ObjectToBytes(string s3ObjectName) { try { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(s3ObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); byte[] buffer = new byte[1024]; using (MemoryStream ms = new MemoryStream()) { int read; while ((read = response.ResponseStream.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, read); } return ms.ToArray(); } } catch (Exception e) { structuredLog("E", "Exception in DownloadS3ObjectToBytes: " + e); return null; } }