protected void Page_Load(object sender, EventArgs e) { AWSAuthConnection conn = new AWSAuthConnection(accessKey, secretKey); XmlTextReader r = new XmlTextReader(Request.InputStream); r.MoveToContent(); string xml = r.ReadOuterXml(); XmlDocument documentToSave = new XmlDocument(); documentToSave.LoadXml(xml); SortedList metadata = new SortedList(); metadata.Add("title", bucket); metadata.Add("Content-Type", "application/xml"); S3Object titledObject = new S3Object(documentToSave.OuterXml, metadata); SortedList headers = new SortedList(); headers.Add("Content-Type", "application/xml"); headers.Add("x-amz-acl", "public-read"); conn.put(bucket, documentKey, titledObject, headers); Response.Write("saved: " + documentToSave.OuterXml); }
public WasabiRemoteItem(S3Object sourceItem, S3Updater source, string rootFolder) { _source = source ?? throw new ArgumentNullException(nameof(source)); _sourceItem = sourceItem ?? throw new ArgumentNullException(nameof(sourceItem)); _fullPath = _sourceItem.Key.TrimEnd('/'); if (rootFolder != null) { _rootFolder = rootFolder.TrimEnd('/'); if (!_fullPath.StartsWith(_rootFolder)) { throw new IOException($"Remote item full path {_fullPath} doesn't start with the specified root path {_rootFolder}"); } ClientRelativeFileName = _fullPath.Substring(_rootFolder.Length).Trim('/'); } IsDirectory = IsDirectory(sourceItem); if (IsDirectory) { throw new ArgumentException("Directory object received in wrong overload"); } IsFile = true; ItemSize = _sourceItem.Size; ModifiedTime = _sourceItem.LastModified; Name = Path.GetFileName(_fullPath); }
private void ProcessFile(S3Object file) { Trace.WriteLine(string.Format("Processing item {0}", file.Key)); var lines = ReadS3File(file); var trimmedDate = file.Key.Substring(DATE_EXTRACTION_PREFIX.Length, 11); var date = DateTime.ParseExact(trimmedDate, FORMAT, PROVIDER); var fileMod = file.GetHashCode() % ComputeNode.GlobalBucketCount; var buckets = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName == CATALOG).First().Buckets; var bucketMod = buckets.First(b => b.Value.BucketMod == fileMod).Value; Trace.WriteLine(string.Format("Adding data items from {0}", file.Key)); lines.AsParallel().ForAll(line => { var items = line.Split(' '); Debug.Assert(items.Length == 4); var projectCode = HttpUtility.UrlDecode(items[0]); var pageName = HttpUtility.UrlDecode(items[1]); var pageViews = int.Parse(items[2]); var pageSizeKB = long.Parse(items[3]); var wikiStat = new WikipediaHourlyPageStats(date, projectCode, pageName, pageViews, pageSizeKB); bucketMod.BucketDataTables[TABLE].AddItem(wikiStat); }); Trace.WriteLine(string.Format("Added data items from {0}", file.Key)); }
private static void WriteToBlobStorage(BucketBlobPackage package) { foreach (var item in package.FileInfo) { using (client = new AmazonS3Client(package.AccessKey, package.SecretAccessKey, GetAwsRegionFromString(package.BucketInfo.Region))) { GetObjectRequest request = new GetObjectRequest { BucketName = package.BucketInfo.Name, Key = item.Key }; using (GetObjectResponse response = client.GetObject(request)) { using (Stream responseStream = response.ResponseStream) { string fileExtension = item.Key.Substring(Math.Max(0, item.Key.Length - 4)).ToUpper(); switch (fileExtension) { case ".ZIP": CopyZIPtoBlob(responseStream); break; default: S3Object o = item.Value as S3Object; CopySingleFileToBlob(responseStream, item.Key, o.Size); break; } } } } } }
/// <summary> /// Download a specific file from a S3 bucket /// </summary> /// <param name="path"></param> public void DownloadFile(string path) { TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(accessKey, secretKey, Amazon.RegionEndpoint.USEast1)); BasicAWSCredentials basicCredentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), Amazon.RegionEndpoint.USEast1); ListObjectsRequest request = new ListObjectsRequest(); request.BucketName = bucketName; ListObjectsResponse response = s3Client.ListObjects(request.BucketName = bucketName, request.Prefix = path); try { S3Object obj = response.S3Objects[0]; string filename = filepath + "\\" + obj.Key; FileStream fs = File.Create(filename); fs.Close(); fileTransferUtility.Download(filename, bucketName, obj.Key); log.Info("File: " + obj.Key + " downloaded"); } catch (Exception ex) { log.Info(ex.Message + ex.StackTrace); } }
//path format (band/16/profile.jpg) or (profile.jpg) public async Task <bool> DeleteFile(string filepath) { try { bool status = false; //int pos = filepath.LastIndexOf("/") + 1; //string path = filepath.Substring(pos, filepath.Length) var list = await s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = BucketName, Prefix = filepath }); S3Object s3Object = list.S3Objects.FirstOrDefault(); if (s3Object != null) { var res = await s3Client.DeleteObjectAsync(new DeleteObjectRequest() { BucketName = BucketName, Key = s3Object.Key }); status = true; } return(status); } catch (Exception e) { Console.WriteLine(e.Message); return(false); } }
private bool ShouldDownload(S3Object s3o) { // skip objects based on ModifiedSinceDateUtc if (this._request.IsSetModifiedSinceDateUtc() && s3o.LastModified.ToUniversalTime() <= this._request.ModifiedSinceDateUtc.ToUniversalTime()) { return(false); } // skip objects based on UnmodifiedSinceDateUtc if (this._request.IsSetUnmodifiedSinceDateUtc() && s3o.LastModified.ToUniversalTime() > this._request.UnmodifiedSinceDateUtc.ToUniversalTime()) { return(false); } #pragma warning disable CS0618 //A class member was marked with the Obsolete attribute // skip objects based on ModifiedSinceDate if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate) { return(false); } // skip objects based on UnmodifiedSinceDate if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate) { return(false); } #pragma warning restore CS0618 //A class member was marked with the Obsolete attribute // skip objects which are instruction files and we're using encryption client if (IsInstructionFile(s3o.Key)) { return(false); } return(true); }
private SourceImage ProcessObject( Bucket tracked, S3Object obj, IDictionary<string, ImageEntry> entries, IList<ImageEntry> toAdd) { if (entries.TryGetValue(obj.Key, out var entry)) { return this.GetFromCacheEntry(entry); } else { var image = this.CreateSourceImage( key: obj.Key, lastModified: obj.LastModified, totalBytes: obj.Size); toAdd.Add(new ImageEntry() { Bucket = tracked, Key = obj.Key, LastModified = obj.LastModified, FileSize = obj.Size, Status = image.Status == ImageStatus.Skipped ? ImageStatus.Pending : image.Status }); return image; } }
// Download object to local file // Returns true on success, false on error. public override bool DownloadFile(CloudFile file, String outputFilePath) { try { this.Exception = null; int pos = outputFilePath.LastIndexOf("\\"); if (pos != -1) { outputFilePath = outputFilePath.Substring(0, pos); } S3Object obj = file.StorageObject as S3Object; TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(this.AccessKey, this.SecretKey, this.RegionEndpoint)); fileTransferUtility.Download(outputFilePath, obj.BucketName, file.Name); return(true); } catch (Exception ex) { this.Exception = ex; if (!this.HandleErrors) { throw ex; } return(false); } }
//path format (band/16) or (band) public async Task <bool> DeleteFolder(string path) { try { bool status = false; var list = await s3Client.ListObjectsAsync(new ListObjectsRequest() { BucketName = BucketName, Prefix = path }); S3Object s3Object = list.S3Objects.FirstOrDefault(); if (s3Object == null) { var res = await s3Client.DeleteObjectAsync(new DeleteObjectRequest() { BucketName = BucketName, Key = s3Object.Key }); status = true; } return(status); } catch (Exception e) { Console.WriteLine(e.Message); return(false); } }
public S3FileSystemItem(S3Object file, string key) { this.Name = PathEx.GetFileName(key); this.Size = file.Size; this.IsDirectory = false; this.LastModifyTime = new DateTimeOffset(file.LastModified.ToUniversalTime(), TimeSpan.Zero); }
private bool Backup(S3Object obj, FileObject fo) { string displayName = Helpers.FormatDisplayFileName(_wideDisplay, Helpers.RemoveSecExt(obj.Key)); Logger.WriteLog(ErrorCodes.S3BucketSource_SyncFile, $"cpy src {displayName} [{Helpers.FormatByteSize(fo.DataContent.Length)}]", Severity.Information, VerboseLevel.User); // Transforms the file if a transformation is provided FileObject fo2 = Transform != null?Transform.Process(fo) : fo; bool result = Destination.Write(fo2); if (result) { Interlocked.Increment(ref _fileSynchronized); Interlocked.Add(ref _writeSize, fo2.DataContent.Length); } if (result) { Interlocked.Increment(ref _synchronizedCount); } else { Interlocked.Increment(ref _errorCount); } return(result); }
private S3Object getFile() { AmazonS3Client client = getConfig(); ListObjectsV2Request request = new ListObjectsV2Request() { BucketName = _bucketName }; ListObjectsV2Response list = client.ListObjectsV2(request); long data = 0; S3Object file = new S3Object(); foreach (var item in list.S3Objects) { if (data < item.Size) { file = item; data = item.Size; } } // Convert Bytes to MB double t = (file.Size / 1024f) / 1024f; // Check how many files we need to achieve x quantity of GB (This case 4.9GB) t = 4900 / t; // Round number and converted to Int cantidad = Convert.ToInt32(Math.Round(t)); // Return S3 Object return(file); }
public void Remove_previously_saved_S3_object(string s3ObjectKey, S3Object s3Object = null) { "And a S3 Object with given key is saved" .x(async() => { s3Object = new S3Object { Bucket = BucketId, Key = s3ObjectKey, CreationDate = DateTime.UtcNow, Content = new byte[] { 0x01, 0x02, 0x03 }, ContentType = "application/octet-stream" }; await _storage.AddObjectAsync(s3Object); }); "When method RemoveS3ObjectAsync() will be called with given key" .x(async() => await _storage.DeleteObjectAsync(BucketId, s3ObjectKey)); "Then the corresponding reliable dictionaries are removed" .x(async() => { using (var tx = _stateManager.CreateTransaction()) { var uriPrefix = $"fabric://mocks/{S3EntriesKey}:{s3Object.Bucket}:"; var enumerator = _stateManager.GetAsyncEnumerator(); while (await enumerator.MoveNextAsync(CancellationToken.None)) { var uriStr = enumerator.Current.Name.ToString(); Assert.False(uriStr.StartsWith(uriPrefix)); } } }); }
public S3Object GetS3File() { S3Object s3Object; try { var request = new GetObjectRequest { BucketName = this.BucketName, Key = this.Key }; var response = this.S3Client.GetObject(request); s3Object = new S3Object { ETag = response.ETag, Key = response.Key, LastModified = response.LastModified.ToUniversalTime() }; } catch (AmazonS3Exception aex) { const string FORMAT = "Couldn't get file ({0}:/{1})."; var message = string.Format(FORMAT, this.BucketName, this.Key); throw new Exception(message, aex); } return(s3Object); }
public async Task ListObjects(S3Provider provider, string _, ISimpleClient client) { await CreateTempBucketAsync(provider, client, async tempBucket => { string tempObjName = "object-" + Guid.NewGuid(); PutObjectResponse putResp = await client.PutObjectStringAsync(tempBucket, tempObjName, "hello").ConfigureAwait(false); Assert.Equal(200, putResp.StatusCode); ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket).ConfigureAwait(false); Assert.Equal(200, listResp.StatusCode); Assert.Equal(tempBucket, listResp.BucketName); Assert.Equal(1, listResp.KeyCount); if (provider == S3Provider.AmazonS3) { Assert.Equal(1000, listResp.MaxKeys); } Assert.False(listResp.IsTruncated); S3Object obj = listResp.Objects.First(); Assert.Equal(tempObjName, obj.ObjectKey); Assert.Equal("\"5d41402abc4b2a76b9719d911017c592\"", obj.ETag); Assert.Equal(5, obj.Size); Assert.Equal(DateTime.UtcNow, obj.LastModifiedOn.UtcDateTime, TimeSpan.FromSeconds(5)); if (provider == S3Provider.AmazonS3) { Assert.Equal(StorageClass.Standard, obj.StorageClass); } }).ConfigureAwait(false); }
/// <summary> /// Tracks the specified <see cref="S3Object"/>. /// </summary> /// <param name="o">The <see cref="S3Object"/> being synchronized.</param> /// <param name="copied">Whether or not the item needed to be copied.</param> /// <param name="updated">Whether or not the item needed to be udpated.</param> public void TrackObject(string prefix, S3Object o, bool copied, bool updated) { if (copied) { Interlocked.Exchange(ref _lastCopyCompleted, o.Key); Interlocked.Exchange(ref _lastCopyDate, o.LastModified.Ticks); } if (updated) { Interlocked.Exchange(ref _lastUpdateCompleted, o.Key); Interlocked.Exchange(ref _lastUpdateDate, o.LastModified.Ticks); } InterlockedMax(ref _greatestKeyCompleted, o.Key.Substring(prefix.Length)); InterlockedMin(ref _earliestDate, o.LastModified.Ticks); InterlockedMax(ref _latestDate, o.LastModified.Ticks); Interlocked.Add(ref _bytesProcessed, o.Size); Interlocked.Add(ref _objectsProcessed, 1); if (copied) { InterlockedMin(ref _earliestCopiedDate, o.LastModified.Ticks); InterlockedMax(ref _latestCopiedDate, o.LastModified.Ticks); Interlocked.Add(ref _bytesCopied, o.Size); Interlocked.Add(ref _objectsCopied, 1); } if (updated) { InterlockedMin(ref _earliestUpdatedDate, o.LastModified.Ticks); InterlockedMax(ref _latestUpdatedDate, o.LastModified.Ticks); Interlocked.Add(ref _bytesUpdated, o.Size); Interlocked.Add(ref _objectsUpdated, 1); } }
private Response AddObject(string bucket, string key, Stream stream) { if (Request.Url.Query == "?acl") { return(new Response { StatusCode = HttpStatusCode.OK }); } var content = stream.Copy(configuration.MaxBytesPerSecond); var s3Object = new S3Object { Bucket = bucket, Key = key, ContentType = Request.Headers.ContentType, CreationDate = DateTime.UtcNow, Content = () => content, ContentMD5 = content.GenerateMD5CheckSum(), Size = content.Length }; storage.AddObject(s3Object); var response = new Response { StatusCode = HttpStatusCode.OK }; response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5)); return(response); }
private async Task UpdateItem(S3Object sourceItem, FileInfo targetPath, IProgress <double> progressCallback, CancellationToken cancellationToken) { progressCallback.Report(0d); //_s3Client.DownloadToFilePathAsync() doesn't support progress callbacks, need to do all this instead var objectResponse = await _s3Client.GetObjectAsync(sourceItem.BucketName, sourceItem.Key, cancellationToken); using (var input = objectResponse.ResponseStream) using (var output = targetPath.OpenWrite()) { // buffer based on file size to have decent rate of progress reporting const int preferredUpdateSteps = 100; const long minBufferSize = 128 * 1024; //kb const long maxBufferSize = 5 * 1024 * 1024; //mb var buffer = new byte[Math.Min(maxBufferSize, Math.Max(minBufferSize, sourceItem.Size / preferredUpdateSteps))]; int bytesRead; while ((bytesRead = await input.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0) { var task = output.WriteAsync(buffer, 0, bytesRead, cancellationToken); progressCallback.Report(100d * Math.Min(1d, (double)output.Position / (double)sourceItem.Size)); await task; } if (output.Position != sourceItem.Size) { throw new InvalidDataException("The downloaded file was not the correct size"); } } }
/// <summary> /// Write files to desired destination path. /// </summary> /// <param name="parameters"></param> /// <param name="fileObject"></param> /// <param name="s3Client"></param> /// <param name="destinationFolder"></param> /// <param name="fullPath"></param> /// <returns>string</returns> private static async Task <string> WriteToFile( Parameters parameters, S3Object fileObject, AmazonS3Client s3Client, string destinationFolder, string fullPath ) { string responseBody; var request = new GetObjectRequest { BucketName = parameters.BucketName, Key = fileObject.Key }; using (var response = await s3Client.GetObjectAsync(request)) using (var responseStream = response.ResponseStream) using (var reader = new StreamReader(responseStream)) responseBody = await reader.ReadToEndAsync(); if (File.Exists(fullPath)) { File.Delete(fullPath); File.WriteAllText(fullPath, responseBody); return(fullPath); } else { if (!Directory.Exists(destinationFolder)) { Directory.CreateDirectory(destinationFolder); } File.WriteAllText(fullPath, responseBody); return(fullPath); } }
protected override async Task ExecuteAsync(CommandLineApplication app, CancellationToken token) { IAsyncEnumerator <S3Object> list = Manager.ObjectManager.ListAsync(BucketName, IncludeOwner).GetAsyncEnumerator(token); bool hasData = await list.MoveNextAsync().ConfigureAwait(false); if (!hasData) { Console.WriteLine(); Console.WriteLine("There were no objects."); } else { Console.WriteLine(); if (IncludeDetails) { if (IncludeOwner) { Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4,-20}{5}", "Modified on", "Size", "Storage class", "ETag", "Owner", "Name"); } else { Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4}", "Modified on", "Size", "Storage class", "ETag", "Name"); } } else { Console.WriteLine("{0,-20}{1,-12}{2}", "Modified on", "Size", "Name"); } do { S3Object obj = list.Current; if (IncludeDetails) { if (IncludeOwner) { string ownerInfo = string.Empty; if (obj.Owner != null) { ownerInfo = obj.Owner.Name; } Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4,-20}{5}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.StorageClass, obj.ETag !, ownerInfo, obj.ObjectKey); } else { Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.StorageClass, obj.ETag !, obj.ObjectKey); } } else { Console.WriteLine("{0,-20}{1,-12}{2}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.ObjectKey); } } while (await list.MoveNextAsync().ConfigureAwait(false)); } }
public async Task <IActionResult> Put(string bucket, string key, CancellationToken cancellationToken) { if (Request.QueryString.HasValue && Request.QueryString.Value == "?acl") { return(Ok()); } var s3Object = new S3Object { Bucket = bucket, Key = key, ContentType = string.IsNullOrEmpty(Request.ContentType) ? "application/octet-stream" : Request.ContentType, CreationDate = DateTime.UtcNow, Size = 0 }; if (Request.Body != null) { s3Object.Content = await Request.Body.ReadAllBytesAsync(); } await _storage.AddObjectAsync(s3Object, cancellationToken); var response = Ok(); Response.Headers.Add("ETag", $"\"{s3Object.ContentMD5}\""); return(response); }
/// <summary> /// Downloads the latest backup set from the Aws service. /// </summary> /// <param name="compressor">The compresor to use when decompressing the downloaded file.</param> /// <returns>The path to the downloaded and decompressed backup file.</returns> public string DownloadBackup(ICompressor compressor) { S3Object latest = this.GetLatestBackupItem(); string path = latest.Key; if (!String.IsNullOrEmpty(this.Target.AwsPrefix) && path.StartsWith(this.Target.AwsPrefix, StringComparison.OrdinalIgnoreCase)) { path = path.Substring(this.Target.AwsPrefix.Length); if (path.StartsWith("/", StringComparison.Ordinal)) { path = path.Substring(1); } } path = Path.Combine(TempDir, path); string fileName = Path.GetFileName(path); if (File.Exists(path)) { File.Delete(path); } TransferInfo info = new TransferInfo() { BytesTransferred = 0, FileName = fileName, FileSize = 0 }; using (TransferUtility transfer = new TransferUtility(S3Client)) { TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest() .WithBucketName(AwsConfig.BucketName) .WithFilePath(path) .WithKey(latest.Key); request.WriteObjectProgressEvent += (sender, e) => { info.BytesTransferred = e.TransferredBytes; info.FileSize = e.TotalBytes; this.Fire(this.TransferProgress, new TransferInfo(info)); }; this.Fire(this.TransferStart, new TransferInfo(info)); transfer.Download(request); } this.Fire(this.TransferComplete, new TransferInfo(info)); this.Fire(this.DecompressStart); string decompressedPath = compressor.Decompress(path); this.Fire(this.DecompressComplete); File.Delete(path); return(decompressedPath); }
/// <summary> /// Pick up raw ETag string without "" defined in RFC /// </summary> /// <param name="source"></param> /// <returns></returns> public static string GetETag(this S3Object source) { if (source == null) { throw new ArgumentNullException(); } return(source.ETag.Replace("\"", "")); }
private static void UnmarshallResult(XmlUnmarshallerContext context, ListObjectsResponse response) { int currentDepth = context.get_CurrentDepth(); int num = currentDepth + 1; if (context.get_IsStartOfDocument()) { num += 2; } while (context.Read()) { if (context.get_IsStartElement() || context.get_IsAttribute()) { if (context.TestExpression("IsTruncated", num)) { response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("NextMarker", num)) { response.NextMarker = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Contents", num)) { S3Object s3Object = ContentsItemUnmarshaller.Instance.Unmarshall(context); s3Object.BucketName = response.Name; response.S3Objects.Add(s3Object); } else if (context.TestExpression("Name", num)) { response.Name = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Prefix", num)) { response.Prefix = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("Delimiter", num)) { response.Delimiter = StringUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("MaxKeys", num)) { response.MaxKeys = IntUnmarshaller.GetInstance().Unmarshall(context); } else if (context.TestExpression("CommonPrefixes", num)) { string text = CommonPrefixesItemUnmarshaller.Instance.Unmarshall(context); if (text != null) { response.CommonPrefixes.Add(text); } } } else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth) { break; } } }
public async Task <S3Response> DownloadFileAsync(string bucketName, string folderName, string FileName) { string target = Path.GetTempPath(); folderName = folderName.Replace(@"\", "/"); try { if (await AmazonS3Util.DoesS3BucketExistV2Async(_client, bucketName) == false) { ListObjectsRequest request = new ListObjectsRequest() { BucketName = bucketName }; do { ListObjectsResponse response = await _client.ListObjectsAsync(request); for (int i = 1; i <= response.S3Objects.Count - 1; i++) { S3Object entry = response.S3Objects[i]; if (entry.Key.Replace(folderName + "/", "") == FileName) { GetObjectRequest objRequest = new GetObjectRequest() { BucketName = bucketName, Key = entry.Key }; GetObjectResponse objResponse = await _client.GetObjectAsync(objRequest); await objResponse.WriteResponseStreamToFileAsync(target + FileName, true, CancellationToken.None); break; } } if ((response.IsTruncated)) { request.Marker = response.NextMarker; } else { request = null /* TODO Change to default(_) if this is not a reference type */; } }while (request != null); } } catch (AmazonS3Exception e) { return(new S3Response { Message = e.Message, Status = e.StatusCode }); } catch (Exception ex) { return(new S3Response { Message = ex.Message, Status = HttpStatusCode.InternalServerError }); } return(new S3Response { Message = "Something went wrong...", Status = HttpStatusCode.InternalServerError }); }
public static string ToString(S3Object s) { if (s == null) { return(""); } return(Path.Combine(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString() + s.BucketName, s.Key.Replace('/', Path.DirectorySeparatorChar)).TrimEnd(Path.DirectorySeparatorChar)); }
public static async Task supprime(S3Object obj, string bucketName, AmazonS3Client amazonS3Client) { DeleteObjectRequest requestDelete = new DeleteObjectRequest { BucketName = bucketName, Key = obj.Key, }; await amazonS3Client.DeleteObjectAsync(requestDelete); }
// return a file from S3 private static S3Object getObject(string filename) { S3Object o = new S3Object() { Bucket = "licenselatesfx", Name = filename }; return(o); }
private static async Task <string> GetFileContent(IAmazonS3 client, S3Object s3Object) { var response = await client.GetObjectAsync(s3Object.BucketName, s3Object.Key); await using var responseStream = response.ResponseStream; using var reader = new StreamReader(responseStream); return(await reader.ReadToEndAsync()); }
// // GET: /S3Object/Edit/5 public ActionResult Edit(string id = null, string prefix = "", int maxKeys = 100) { ViewBag.prefix = prefix; ViewBag.maxKeys = maxKeys; var list = new Amazon.S3.Model.ListObjectsRequest(); list.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]); list.WithPrefix(id); var s3Objects = s3.ListObjects(list).S3Objects; if (s3Objects.Count == 0) { return HttpNotFound(); } var get = new Amazon.S3.Model.GetObjectRequest(); get.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]); get.WithKey(s3Objects[0].Key); var response = s3.GetObject(get); S3Object modelObject = new S3Object { Key = s3Objects[0].Key, Size = s3Objects[0].Size, LastModified = s3Objects[0].LastModified, ContentType = response.ContentType }; return View(modelObject); }
private void DownloadFolder(S3Object s3Obj, string folderPath) { string fullPath = s3Obj.Shared ? folderPath : Path.Combine(folderPath, s3Obj.Key.Replace("/", "\\")); try { // need to import complete directory _downloadObjects.Add(fullPath); if (!Directory.Exists(fullPath)) Directory.CreateDirectory(fullPath); // get the folder structure from application database string url = Utilities.DevelopmentMode ? "http://localhost:3000" : "http://versavault.com"; url += "/api/child_files?bucket_key=" + Utilities.MyConfig.BucketKey + "&parent_uid=" + s3Obj.Uid + "&machine_key=" + Utilities.MyConfig.MachineKey + "&shared=" + s3Obj.Shared; string result = GetResponse(url); if (!string.IsNullOrEmpty(result)) { var res = JsonConvert.DeserializeObject<s3_object>(result); foreach (var s3ObjSub in res.S3Object) { if (s3ObjSub != null) { string fullPathSub = Path.Combine(s3ObjSub.Shared ? Utilities.SharedFolderPath + "\\" + s3ObjSub.Username : Utilities.Path, s3ObjSub.Key.Replace("/", "\\")); if (s3ObjSub.Shared) { if (!Directory.Exists(Utilities.SharedFolderPath + "\\" + s3ObjSub.Username)) Directory.CreateDirectory(Utilities.SharedFolderPath + "\\" + s3ObjSub.Username); } if (s3ObjSub.Folder) { if (!Directory.Exists(fullPathSub)) { DownloadFolder(s3ObjSub, fullPathSub); } else { try { TimeSpan timeSpan = new DirectoryInfo(fullPathSub).LastWriteTime.ToUniversalTime().Subtract(s3ObjSub.LastModified); var seconds = Math.Floor(timeSpan.TotalSeconds); if (seconds != 0) { if (seconds < 0) { _downloadObjects.Add(fullPathSub); DownloadFolder(s3ObjSub, fullPathSub); string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/"); if (s3ObjSub.Shared) { var folders = relativePath.Split('\\'); ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]); } else ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty); //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update }); } //else // Uploadfiles(fullPathSub); // need to figure it out how to do this beacause we cannot upload the entire folder here } } catch (Exception) { // Todo // need to fix this when an error occured continue; } } } else { if (!File.Exists(fullPathSub)) { _downloadObjects.Add(fullPathSub); _service.GetObject(s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey, s3ObjSub.Key, fullPathSub); string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/"); if (s3ObjSub.Shared) { var folders = relativePath.Split('\\'); ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]); } else ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty); //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update }); } else { try { if (!Utilities.IsFileUsedbyAnotherProcess(fullPathSub)) { TimeSpan timeSpan = new FileInfo(fullPathSub).LastWriteTime.ToUniversalTime().Subtract(s3ObjSub.LastModified); var seconds = Math.Floor(timeSpan.TotalSeconds); if (seconds != 0) { if (seconds < 0) { _downloadObjects.Add(fullPathSub); _service.GetObject(s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey, s3ObjSub.Key, fullPathSub); string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/"); if (s3ObjSub.Shared) { var folders = relativePath.Split('\\'); ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]); } else ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty); //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update }); } else Addobject(fullPathSub, s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey); } } } catch (Exception) { // Todo // need to fix this when an error occured continue; } } } } } } } catch (Exception) { // Todo // need to figure it out why it is happening } finally { string relativePath = fullPath.Replace(Utilities.Path + "\\", "").Replace("\\", "/"); if (relativePath.ToLower().IndexOf("shared") == 0) { var folders = relativePath.Split('\\'); ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPath).LastWriteTime, Status = UpdateStatus.Add }, true, folders.Length > 1 ? folders[1] : folders[0]); } else ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPath).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty); //_applicationUpates.Enqueue(new AppUpdateInfo { Key = s3Obj.Key, LastModifiedTime = new DirectoryInfo(fullPath).LastWriteTime, Status = UpdateStatus.Update }); } }
public ActionResult Edit(S3Object s3object, string prefix = "", int maxKeys = 100) { ViewBag.prefix = prefix; ViewBag.maxKeys = maxKeys; if (ModelState.IsValid && Request.Files.Count > 0) { var put = new Amazon.S3.Model.PutObjectRequest(); put.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]); put.WithKey(s3object.Key); var file = Request.Files[0]; put.WithContentType(file.ContentType); put.WithInputStream(file.InputStream); put.WithCannedACL(Amazon.S3.Model.S3CannedACL.PublicRead); s3.PutObject(put); return RedirectToAction("Index", new { prefix = prefix, maxKeys = maxKeys }); } return View(s3object); }