public ActionResult ViewStats(string id) { using (var client = Amazon.AWSClientFactory.CreateAmazonS3Client(AwsAccessKey, AwsSecretAccessKey)) { var request = new GetObjectRequest(); request.WithBucketName("MyTwitterStats"); request.WithKey(id + ".json.gz"); GetObjectResponse response = null; try { response = client.GetObject(request); } catch (AmazonS3Exception) { //TODO: Log exception.ErrorCode return HttpNotFound(); } using (response) using (var stream = response.ResponseStream) using (var zipStream = new GZipStream(stream, CompressionMode.Decompress)) using (var reader = new StreamReader(zipStream)) { var jsonSerializer = JsonSerializer.Create(new JsonSerializerSettings()); var stats = jsonSerializer.Deserialize<Stats>(new JsonTextReader(reader)); return View(stats); } } }
public AmazonS3Object GetFile(string key) { var request = new GetObjectRequest(); request.WithBucketName(this._bucketName).WithKey(key); try { var response = this._client.GetObject(request); return response.ToAmazonS3Object(); } catch (Exception ex) { Log.Error(string.Format("Cannot find the file with key {0}. Debug message: {1}", key, ex.Message)); return null; } }
public static byte[] GetFile(string name) { try { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(BucketName) .WithKey(name); AmazonS3Config config = new AmazonS3Config(); config.CommunicationProtocol = Protocol.HTTP; using (MemoryStream ms = new MemoryStream()) using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(AccessKeyID, SecretAccessKeyID, config)) using (GetObjectResponse response = client.GetObject(request)) { if (response != null) { response.ResponseStream.CopyTo(ms); return ms.ToArray(); } else return null; } } catch (AmazonS3Exception amazonS3Exception) { #if(DEBUG) if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity"))) { Console.WriteLine("Please check the provided AWS Credentials."); Console.WriteLine("If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3"); } else { Console.WriteLine("An error occurred with the message '{0}' when deleting an object", amazonS3Exception.Message); } #endif return null; } catch (Exception e) { return null; } }
public static bool ReadLogFile(string s3FileName, string credentialFilePath) { Type t = System.Reflection.MethodBase.GetCurrentMethod().DeclaringType; try { if (ReadS3Credentials(credentialFilePath) == false) { LogEvents.S3NoCredentials(t); return false; } AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(_accessKeyId, _secretAccessKey); GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(_bucketName).WithKey(s3FileName); S3Response responseWithMetadata = client.GetObject(request); return true; } catch (AmazonS3Exception amazonS3Exception) { LogEvents.S3Error(t, amazonS3Exception); return false; } }
/* Sample call for upload:- byte[] array = new byte[1024*1024*1024]; Random random = new Random(); random.NextBytes(array); double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob"); double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob"); * * */ public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType, SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName=null, string blobName=null) { string accountName = ConfigurationManager.AppSettings.Get("AccountName"); string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); DateTime begin=DateTime.Now, end=DateTime.Now; if (synchronizerType == SynchronizerType.Azure) { #region azure download/upload if (containerName==null) containerName = "testingraw"; if(blobName==null) blobName = Guid.NewGuid().ToString(); CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); if (syncDirection == SynchronizeDirection.Upload) { logger.Log("Start Stream Append"); container.CreateIfNotExist(); begin = DateTime.UtcNow;////////////////////////////////////// try { using (MemoryStream memoryStream = new System.IO.MemoryStream(input)) { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); blockBlob.UploadFromStream(memoryStream); } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Append"); } if (syncDirection == SynchronizeDirection.Download) { logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); try { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); byte[] blobContents = blockBlob.DownloadByteArray(); //if (File.Exists(blobName)) // File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// // using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate)) // { byte[] contents = blockBlob.DownloadByteArray(); // fs.Write(contents, 0, contents.Length); // } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } #endregion } else if (synchronizerType == SynchronizerType.AmazonS3) { #region amazon s3 stuff if (containerName == null) containerName = "testingraw"; if (blobName == null) blobName = Guid.NewGuid().ToString(); AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey); if (syncDirection == SynchronizeDirection.Upload) { ListBucketsResponse response = amazonS3Client.ListBuckets(); foreach (S3Bucket bucket in response.Buckets) { if (bucket.BucketName == containerName) { break; } } amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName)); begin = DateTime.UtcNow;////////////////////////////////////// MemoryStream ms = new MemoryStream(); ms.Write(input, 0, input.Length); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); request.InputStream = ms; amazonS3Client.PutObject(request); end = DateTime.UtcNow;////////////////////////////////////// } if (syncDirection == SynchronizeDirection.Download) { if (File.Exists(blobName)) File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); GetObjectResponse response = amazonS3Client.GetObject(request); var localFileStream = File.Create(blobName); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); end = DateTime.UtcNow;////////////////////////////////////// } #endregion } else { throw new InvalidDataException("syncronizer type is not valid"); } return (end - begin).TotalMilliseconds;// return total time to upload in milliseconds }
private void getAndSaveFile(string keyName, string storedFileName) { GetObjectRequest getRequest = new GetObjectRequest(); getRequest.WithBucketName("intelrecruiter") .WithKey(keyName); using (GetObjectResponse response = client.GetObject(getRequest)) { string dest = Path.Combine(zipDirectoryName, storedFileName); if (!File.Exists(dest)) { response.WriteResponseStreamToFile(dest); } } }
public byte[] UploadFileAsChunks(string filePath) { string s3objectName; List<ChunkInfo> chunkList_cloud = new List<ChunkInfo>(); ; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....) List<ChunkInfo> chunkList_local; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....) try { if (logger != null) logger.Log("Start Synchronizer Check Blob Exists"); s3objectName = Path.GetFileName(filePath); bool s3ObjectExists = S3ObjectExists(ChunkMetadataObjectPrefix + s3objectName); if (logger != null) logger.Log("End Synchronizer Check Blob Exists"); if (s3ObjectExists) { if (logger != null) logger.Log("Start Synchronizer Fill Remote ChunkList"); GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(ChunkMetadataObjectPrefix + s3objectName); GetObjectResponse response = amazonS3Client.GetObject(request); StreamReader reader = new StreamReader(response.ResponseStream); string chunkMD_JSON = reader.ReadToEnd(); FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON); StaticChunkSize = fileMD.StaticChunkSize; chunkList_cloud = fileMD.ChunkList; if (logger != null) logger.Log("End Synchronizer Fill Remote ChunkList"); chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType); chunkEncryptionType = SyncFactory.GetEncryptionType(fileMD.encryptionType); } if (logger != null) logger.Log("Start Synchronizer Fill Local ChunkList"); StaticChunk staticChunker = new StaticChunk(StaticChunkSize); chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface // structuredLog("I", "Number of chunks locally: " + chunkList_local.Count); if (logger != null) logger.Log("End Synchronizer Fill Local ChunkList"); if (logger != null) logger.Log("Start Synchronizer ChunkList Compare"); List<ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud); // structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count); // structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count); if (logger != null) logger.Log("End Synchronizer ChunkList Compare"); if (logger != null) logger.Log("Start Synchronizer Upload Multiple Chunks"); UploadChunkList(ref chunkList_toUpload, filePath, s3objectName); if (logger != null) logger.Log("End Synchronizer Upload Multiple Chunks"); // structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count); if (logger != null) logger.Log("Start Synchronizer ChunkList Upload"); string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter()); if (chunkList_toUpload.Count > 0) //upload new chunk list only if we uploaded some new chunks { UploadStringToS3Object(ChunkMetadataObjectPrefix + s3objectName, json); } SHA1 sha1 = new SHA1CryptoServiceProvider(); byte[] ret = sha1.ComputeHash(Encoding.ASCII.GetBytes(json)); if (logger != null) logger.Log("End Synchronizer ChunkList Upload"); return ret; } catch (Exception e) { structuredLog("E", " . UploadFileAsChunks: " + e); return null; } }
public System.Tuple<List<ChunkInfo>, byte[]> GetObjectMetadata(string s3objectName) { List<ChunkInfo> retVal = null; byte[] hash = null; string metadataObjectName = ChunkMetadataObjectPrefix + s3objectName; bool s3ObjectExists = S3ObjectExists(metadataObjectName); if (s3ObjectExists) { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(metadataObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); StreamReader reader = new StreamReader(response.ResponseStream); string chunkMD_JSON = reader.ReadToEnd(); FileMD fileMD = JsonConvert.DeserializeObject<FileMD>(chunkMD_JSON); SHA1 sha1 = new SHA1CryptoServiceProvider(); hash = sha1.ComputeHash(Encoding.ASCII.GetBytes(chunkMD_JSON)); retVal = fileMD.ChunkList; } return new System.Tuple<List<ChunkInfo>, byte[]>(retVal, hash); }
public bool DownloadS3ObjectToFile(string s3ObjectName, string filePath) { try { if (!S3ObjectExists(ChunkMetadataObjectPrefix + s3ObjectName)) return false; GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(s3ObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); if (File.Exists(filePath)) File.Delete(filePath); var localFileStream = File.Create(filePath); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); return true; } catch (Exception e) { structuredLog("E", "Exception in DownloadS3ObjectToFile: " + e); return false; } }
public static void objectSerial(String filePath) { System.Console.WriteLine("\nhello,object!!"); NameValueCollection appConfig = ConfigurationManager.AppSettings; AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"]); String bucketName = "chttest3"; String objectName = "hello"; //String filePath = "D:\\Visual Studio Project\\TestNetSDK\\TestNetSDK\\pic.jpg"; //PutBucket System.Console.WriteLine("PutBucket: {0}\n",bucketName); PutBucketResponse response = s3Client.PutBucket(new PutBucketRequest().WithBucketName(bucketName)); //PutObject System.Console.WriteLine("PutObject!\n"); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(bucketName); request.WithKey(objectName); request.WithFilePath(filePath); PutObjectResponse PutResult = s3Client.PutObject(request); System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult.ETag); //HeadObject System.Console.WriteLine("HeadObject!\n"); GetObjectMetadataResponse HeadResult = s3Client.GetObjectMetadata(new GetObjectMetadataRequest().WithBucketName(bucketName).WithKey(objectName)); System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult.ContentLength,HeadResult.ETag); //GetObject System.Console.WriteLine("GetObject!\n"); GetObjectResponse GetResult = s3Client.GetObject(new GetObjectRequest().WithBucketName(bucketName).WithKey(objectName).WithByteRange(1,15)); Stream responseStream = GetResult.ResponseStream; StreamReader reader = new StreamReader(responseStream); System.Console.WriteLine("Get Object Content:\n {0}\n", reader.ReadToEnd()); System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult.ETag); //CopyObject CopyObjectResponse CopyResult = s3Client.CopyObject(new CopyObjectRequest().WithSourceBucket(bucketName).WithSourceKey(objectName).WithDestinationBucket(bucketName).WithDestinationKey("hihi")); System.Console.WriteLine("CopyObject: ETag: {0} \n",CopyResult.ETag); //DeleteObject System.Console.WriteLine("Delete Object!\n"); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName)); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi")); //copied object //==============================jerry add ============================== System.Console.WriteLine("Jerry Add!\n"); String objectName2 = "hello2"; NameValueCollection nvc = new NameValueCollection(); nvc.Add("A", "ABC"); System.Console.WriteLine("PutObject!\n"); PutObjectRequest request2 = new PutObjectRequest(); request2.WithBucketName(bucketName); request2.WithKey(objectName2); request2.WithAutoCloseStream(true); request2.WithCannedACL(S3CannedACL.BucketOwnerFullControl); request2.WithContentBody("test"); request2.WithContentType("test/xml"); request2.WithGenerateChecksum(true); request2.WithMD5Digest("CY9rzUYh03PK3k6DJie09g=="); request2.WithMetaData(nvc); request2.WithWebsiteRedirectLocation("http://hicloud.hinet.net/"); PutObjectResponse PutResult2 = s3Client.PutObject(request2); System.Console.WriteLine("Uploaded Object Etag: {0}\n", PutResult2.ETag); System.Console.WriteLine("GetObject!\n"); GetObjectRequest request3 = new GetObjectRequest(); request3.WithBucketName(bucketName); request3.WithKey(objectName2); request3.WithByteRange(1, 2); DateTime datetime = DateTime.UtcNow; // 似乎不支援 datetime request3.WithModifiedSinceDate(datetime.AddHours(-1)); request3.WithUnmodifiedSinceDate(datetime.AddHours(1)); request3.WithETagToMatch(PutResult2.ETag); request3.WithETagToNotMatch("notMatch"); GetObjectResponse GetResult2 = s3Client.GetObject(request3); Stream responseStream2 = GetResult2.ResponseStream; StreamReader reader2 = new StreamReader(responseStream2); System.Console.WriteLine("Get Object Content(es):\n {0}\n", reader2.ReadToEnd()); System.Console.WriteLine("Get Object ETag:\n {0}\n", GetResult2.ETag); System.Console.WriteLine("HeadObject!\n"); GetObjectMetadataRequest request4 = new GetObjectMetadataRequest(); request4.WithBucketName(bucketName); request4.WithKey(objectName2); DateTime datetime2 = DateTime.UtcNow; // 似乎不支援 datetime request4.WithModifiedSinceDate(datetime2.AddHours(-1)); request4.WithUnmodifiedSinceDate(datetime2.AddHours(1)); request4.WithETagToMatch(PutResult2.ETag); request4.WithETagToNotMatch("notMatch"); GetObjectMetadataResponse HeadResult2 = s3Client.GetObjectMetadata(request4); System.Console.WriteLine("HeadObject: (1)ContentLength: {0} (2)ETag: {1}\n", HeadResult2.ContentLength, HeadResult2.ETag); CopyObjectRequest request5 = new CopyObjectRequest(); request5.WithSourceBucket(bucketName); request5.WithSourceKey(objectName2); request5.WithDestinationBucket(bucketName); request5.WithDestinationKey("hihi2"); DateTime datetime3 = DateTime.UtcNow; // 似乎不支援 datetime request5.WithModifiedSinceDate(datetime3.AddHours(-1)); request5.WithUnmodifiedSinceDate(datetime3.AddHours(1)); request5.WithETagToMatch(PutResult2.ETag); request5.WithETagToNotMatch("notMatch"); request5.WithMetaData(nvc); request5.WithCannedACL(S3CannedACL.PublicRead); request5.WithContentType("test/xml"); request5.WithWebsiteRedirectLocation("http://hicloud.hinet.net/"); CopyObjectResponse CopyResult2 = s3Client.CopyObject(request5); System.Console.WriteLine("CopyObject: ETag: {0} \n", CopyResult2.ETag); System.Console.WriteLine("Delete Object!\n"); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectName2)); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey("hihi2")); //copied object //==============================jerry add end============================== //DeleteBucket System.Console.WriteLine("Delete Bucket!\n"); //s3Client.DeletesBucket(new DeleteBucketRequest().WithBucketName(bucketName)); s3Client.DeleteBucket(new DeleteBucketRequest().WithBucketName(bucketName)); System.Console.WriteLine("END!"); }
private void Download(string localFolderName, ProcessItem file) { using (var client = CreateAmazonS3Client()) { file.Action = ProcessAction.DownloadingException; if (client == null) return; try { var request = new GetObjectRequest(); request.WithBucketName(_bucketName).WithKey(file.S3Path); using (var response = client.GetObject(request)) { var fileName = ""; if (response.Metadata != null && response.Metadata.Count > 0) { var localPath = response.Metadata.Get("x-amz-meta-localpath"); if (String.IsNullOrWhiteSpace(localPath)) return; localPath = HttpUtility.UrlDecode(localPath); if (localPath != null) { var localPathArr = localPath.Split('\\'); var rootDirectory = localFolderName; for (var i = 1; i < localPathArr.Length - 1; i++) { rootDirectory = Path.Combine(rootDirectory, localPathArr[i]); if(!Directory.Exists(rootDirectory)) Directory.CreateDirectory(rootDirectory); } fileName = Path.Combine(rootDirectory, localPathArr[localPathArr.Length - 1]); } } if (String.IsNullOrWhiteSpace(fileName)) return; using (var fileStream = new FileStream(fileName, FileMode.Create)) { using (var stream = response.ResponseStream) { var data = new byte[32768]; int bytesRead; do { bytesRead = stream.Read(data, 0, data.Length); fileStream.Write(data, 0, bytesRead); } while (bytesRead > 0); fileStream.Flush(); file.Action = ProcessAction.DownloadingDone; } } } } catch (Exception) { } } }
private void getLastSession() { string cookieName; string cookieValue; using (client = Amazon.AWSClientFactory.CreateAmazonS3Client("AKIAJ47VSG7WMA62WLCA", "3tqlHujlftpk6j/z5OtDw2eg9N2FJtz1RwL8bEa3")) { GetObjectRequest txtRequest = new GetObjectRequest(); txtRequest.WithBucketName("intelrecruiter") .WithKey(nsbeCookieValueKey); var response = client.GetObject(txtRequest); StreamReader reader = new StreamReader(response.ResponseStream); cookieValue = reader.ReadToEnd(); response.Dispose(); txtRequest = new GetObjectRequest(); txtRequest.WithBucketName("intelrecruiter") .WithKey(nsbeCookieNameKey); response = client.GetObject(txtRequest); reader = new StreamReader(response.ResponseStream); cookieName = reader.ReadToEnd(); response.Dispose(); } nsbeCookie = new Cookie(cookieName, cookieValue, "/"); }
public string GetObjectInformation(string bucketName, string objectKey) { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName).WithKey(objectKey); GetObjectResponse response = _client.GetObject(request); return response.AmazonId2; }
public byte[] DownloadS3ObjectToBytes(string s3ObjectName) { try { GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(bucketName); request.WithKey(s3ObjectName); GetObjectResponse response = amazonS3Client.GetObject(request); byte[] buffer = new byte[1024]; using (MemoryStream ms = new MemoryStream()) { int read; while ((read = response.ResponseStream.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, read); } return ms.ToArray(); } } catch (Exception e) { structuredLog("E", "Exception in DownloadS3ObjectToBytes: " + e); return null; } }