public void RefreshAssetList()
{
string SecretKey = null;
string PublicKey = null;
AmazonS3Client Client = new AmazonS3Client(PublicKey, SecretKey);
ListObjectsRequest Request = new ListObjectsRequest
{
BucketName = "assets.minecraft.net",
};
ListObjectsResponse Result;
List<Release> releases = new List<Release>();
do
{
Result = Client.ListObjects(Request);
foreach (S3Object o in Result.S3Objects)
{
string IsSnapshot = "Release";
if(!o.Key.Contains("minecraft.jar")) continue;
if (Regex.IsMatch(o.Key, "[0-9][0-9]w[0-9][0-9]")) IsSnapshot = "Snapshot";
else if (o.Key.Contains("pre")) IsSnapshot = "Pre-release";
releases.Add(new Release { Version = o.Key.Split('/')[0],
Size = (o.Size / 1024).ToString() + "KB",
Uploaded = DateTime.Parse(o.LastModified),
Type = IsSnapshot,
Key = o.Key} );
}
}
while (Result.IsTruncated);
releases.Sort(new Comparison<Release>((x, y) => DateTime.Compare(y.Uploaded, x.Uploaded)));
_Releases.Clear();
foreach (Release r in releases)
{
_Releases.Add(r);
}
Client.Dispose();
Result.Dispose();
}
public static IEnumerable<string> ListFiles(string bucketName, string accessKeyID, string secretAccessKey)
{
List<string> fileNames = new List<string>();
try
{
var s3Client = new AmazonS3Client(accessKeyID, secretAccessKey, Amazon.RegionEndpoint.USEast1);
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName
};
while (request != null)
{
ListObjectsResponse response = s3Client.ListObjects(request);
foreach (S3Object entry in response.S3Objects)
{
fileNames.Add(entry.Key);
}
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
}
}
catch (Exception e)
{
}
return fileNames;
}
public override void SetupInitial() {
// This is a chance to do heavy stuff, but in a thread
// Obtain a full list of the bucket. Helps a lot later
ListObjectsRequest listRequest = new ListObjectsRequest() {
BucketName = this.bucket,
Prefix = this.prefix
};
ListObjectsResponse listResponse;
this.files = new List<string>();
this.folders = new List<string>();
// This needs fixing, but somewhere else
this.folders.Add("");
try {
do {
listResponse = client.ListObjects(listRequest);
foreach (S3Object obj in listResponse.S3Objects) {
if (obj.Key.EndsWith("/") && obj.Size == 0)
this.folders.Add(obj.Key.TrimEnd(new char[] { '/' }).Substring(this.prefix.Length));
else
this.files.Add(obj.Key.Substring(this.prefix.Length));
}
listRequest.Marker = listResponse.NextMarker;
} while (listResponse.NextMarker != null);
}
catch (AmazonS3Exception ex) {
string message = ex.Message;
// Clarify this error
if (ex.ErrorCode == "SignatureDoesNotMatch")
message += "\n\nThis can be caused by an invalid S3 secret key credential.";
throw new IOException(message);
}
catch (System.Net.WebException e) { throw new IOException(e.Message); }
}
/// <summary>Determine if our file already exists in the specified S3 bucket</summary>
/// <returns>True if the file already exists in the specified bucket</returns>
public bool FileExists(string fileKey)
{
bool retVal = false;
using (Client)
{
try
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = BucketName
};
using (var response = Client.ListObjects(request))
{
foreach (var file in response.S3Objects)
{
//Project.Log(Level.Info, "File: " + file.Key);
if (file.Key.Equals(fileKey))
{
retVal = true;
}
}
}
}
catch (AmazonS3Exception ex)
{
ShowError(ex);
}
}
return retVal;
}
public static async Task<IList<S3Object>> ListAllObjectsAsync(
this AmazonS3 client, ListObjectsRequest request, CancellationToken ct)
{
string token = null;
var result = new List<S3Object>();
do
{
var req = new ListObjectsRequest
{
BucketName = request.BucketName,
Delimiter = request.Delimiter,
InputStream = request.InputStream,
MaxKeys = request.MaxKeys,
Prefix = request.Prefix,
Marker = token
};
using (ListObjectsResponse response = await client.ListObjectsAsync(req).ConfigureAwait(false))
{
result.AddRange(response.S3Objects);
token = response.NextMarker;
}
} while (token != null && !ct.IsCancellationRequested);
return result;
}
/// <summary>Determine if our file already exists in the specified S3 bucket</summary>
/// <returns>True if the file already exists in the specified bucket</returns>
public bool FileExists(string fileKey)
{
using (Client)
{
try
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = BucketName
};
using (var response = Client.ListObjects(request))
{
foreach (var file in response.S3Objects)
{
if (file.Key.Equals(fileKey))
{
return true;
}
}
}
}
catch (AmazonS3Exception ex)
{
ShowError(ex);
}
}
return false;
}
public void TestCleanup()
{
try
{
var objRequest = new ListObjectsRequest()
{
BucketName = this.bucketName
};
using (var objResponse = client.ListObjects(objRequest))
{
var delRequest = new DeleteObjectsRequest()
{
BucketName = this.bucketName,
Quiet = true
};
delRequest.AddKeys(objResponse.S3Objects.Select(o => new KeyVersion(o.Key)).ToArray());
using (var delResponse = client.DeleteObjects(delRequest))
{
}
}
var deleteRequest = new DeleteBucketRequest()
{
BucketName = this.bucketName
};
using (var deleteResponse = client.DeleteBucket(deleteRequest)) { }
}
catch (Exception ex)
{
this.TestContext.WriteLine("Warning: Could not cleanup bucket: {0}. {1}", this.bucketName, ex);
}
}
public GitBinFileInfo[] ListFiles()
{
var remoteFiles = new List<GitBinFileInfo>();
var client = GetClient();
var listRequest = new ListObjectsRequest();
listRequest.BucketName = _bucketName;
ListObjectsResponse listResponse;
do {
listResponse = client.ListObjects(listRequest);
if (listResponse.S3Objects.Any())
{
var keys = listResponse.S3Objects.Select(o => new GitBinFileInfo(o.Key, o.Size));
remoteFiles.AddRange(keys);
listRequest.Marker = remoteFiles[remoteFiles.Count - 1].Name;
}
}
while (listResponse.IsTruncated);
return remoteFiles.ToArray();
}
public async Task<IReadOnlyDictionary<string, string>> ListAsync(string name, CancellationToken cancellationToken)
{
var files = new Dictionary<string, string>();
var request = new ListObjectsRequest
{
BucketName = _pathManager.Bucket,
Prefix = _pathManager.GetTreeNamePrefix(name)
};
for (;;)
{
var response = await AmazonS3.ListObjectsAsync(request, cancellationToken).ConfigureAwait(false);
foreach (var s3Object in response.S3Objects)
{
var key = _pathManager.GetKeyFromTreeNamePath(name, s3Object.Key);
files[key] = s3Object.ETag;
}
if (!response.IsTruncated)
return files;
request.Marker = response.NextMarker;
}
}
//public ActionResult Save() {
// var path = Request.QueryString["image"];
// if (!string.IsNullOrEmpty(path)) {
// using(var client = new WebClient()) {
// using(var stream = new MemoryStream(client.DownloadData(new Uri(path)))) {
// var request = new PutObjectRequest();
// request.WithBucketName(_settings.BucketName)
// .WithCannedACL(S3CannedACL.PublicRead)
// .WithContentType(MimeMapping.GetMimeMapping(path))
// .WithKey("Images/" + Request.QueryString["title"] + Path.GetExtension(path)).InputStream = stream;
// _client.PutObject(request);
// }
// }
// }
// return View();
//}
public ActionResult Index()
{
ViewBag.Class = "dashboard";
var request = new ListObjectsRequest { BucketName = _settings.BucketName };
var model = _client.ListObjects(request);
return View(model.S3Objects);
}
public string[] ListNames()
{
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = BucketName;
// removed: using() / mh
ListObjectsResponse response = _client.ListObjects(request);
return response.S3Objects.Select(s3o => s3o.Key).ToArray();
}
public void TestAllFilesExistAndModifiedInOutput()
{
try
{
Init();
ListObjectsRequest inputFileObjects;
String fileKey = null;
inputFileObjects = new ListObjectsRequest
{
BucketName = DataTransformer.InputBucketName
};
ListObjectsResponse listResponse;
do
{
listResponse = DataTransformer.s3ForStudentBuckets.ListObjects(inputFileObjects);
foreach (S3Object obj in listResponse.S3Objects)
{
fileKey = obj.Key;
GetObjectRequest request = new GetObjectRequest()
{
BucketName = DataTransformer.OutputBucketName,
Key = fileKey
};
GetObjectResponse response = DataTransformer.s3ForStudentBuckets.GetObject(request);
StreamReader reader = new StreamReader(response.ResponseStream);
string content = reader.ReadToEnd();
if(!content.Contains(DataTransformer.JsonComment))
{
Assert.Fail("Failure - Input file not transformed; output file does not contain JSON comment. " + fileKey);
}
}
// Set the marker property
inputFileObjects.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
}
catch (AmazonServiceException ase)
{
Console.WriteLine("Error Message: " + ase.Message);
Console.WriteLine("HTTP Status Code: " + ase.StatusCode);
Console.WriteLine("AWS Error Code: " + ase.ErrorCode);
Console.WriteLine("Error Type: " + ase.ErrorType);
Console.WriteLine("Request ID: " + ase.RequestId);
}
catch (AmazonClientException ace)
{
Console.WriteLine("Error Message: " + ace.Message);
}
}
public bool StateExists(string appId, string key)
{
var listObjectsRequest = new ListObjectsRequest
{
BucketName = ConfigurationManager.AppSettings["BucketName"],
Prefix = string.Format("{0}/{1}", appId, key),
};
var response = WebApiApplication.AmazonS3Client.ListObjects(listObjectsRequest);
return response.S3Objects.Count == 1;
}
/// <summary>
/// Executes the task.
/// </summary>
/// <returns>True if the task executed successfully, false otherwise.</returns>
public override bool Execute()
{
ListObjectsRequest request = new ListObjectsRequest()
.WithBucketName(BucketName)
.WithPrefix(this.Prefix)
.WithMaxKeys(1);
using (ListObjectsResponse response = Client.ListObjects(request))
{
Exists = response.S3Objects != null && response.S3Objects.Count > 0;
}
return true;
}
private static void ExportAndImport(string folder,CloudBlobContainer container, AmazonS3 s3)
{
var listRequest = new ListObjectsRequest{
BucketName = ConfigurationManager.AppSettings["S3Bucket"],
}.WithPrefix(folder);
Console.WriteLine("Fetching all S3 object in " + folder);
var s3response = s3.ListObjects(listRequest);
//Checking if container exists, and creating it if not
if (container.CreateIfNotExists()) {
Console.WriteLine("Creating the blob container");
}
foreach (var s3Item in s3response.S3Objects)
{
if (s3Item.Key == folder) {
continue;
}
if (s3Item.Key.EndsWith("/"))
{
ExportAndImport(s3Item.Key, container, s3);
continue;
}
Console.WriteLine("---------------------------------------------------");
var blockBlob = container.GetBlockBlobReference(s3Item.Key);
Console.WriteLine("Blob: " + blockBlob.Uri.AbsoluteUri);
var id = blockBlob.StartCopyFromBlob(new Uri("http://" + awsServiceUrl + "/" + s3Bucket + "/" + HttpUtility.UrlEncode(s3Item.Key)), null, null, null);
bool continueLoop = true;
while (continueLoop && id == string.Empty)
{
var copyState = blockBlob.CopyState;
if (copyState != null)
{
var percentComplete = copyState.BytesCopied / copyState.TotalBytes;
Console.WriteLine("Status of blob copy...." + copyState.Status + " " + copyState.TotalBytes + " of " + copyState.BytesCopied + "bytes copied. " + string.Format("{0:0.0%}", percentComplete));
if (copyState.Status != CopyStatus.Pending)
{
continueLoop = false;
}
}
System.Threading.Thread.Sleep(1000);
}
}
}
public async Task DeleteContainerAsync(string containerName)
{
var objectsRequest = new ListObjectsRequest
{
BucketName = _bucket,
Prefix = containerName,
MaxKeys = 100000
};
var keys = new List<KeyVersion>();
try
{
do
{
var objectsResponse = await _s3Client.ListObjectsAsync(objectsRequest);
keys.AddRange(objectsResponse.S3Objects
.Select(x => new KeyVersion() { Key = x.Key, VersionId = null }));
// If response is truncated, set the marker to get the next set of keys.
if (objectsResponse.IsTruncated)
{
objectsRequest.Marker = objectsResponse.NextMarker;
}
else
{
objectsRequest = null;
}
} while (objectsRequest != null);
if (keys.Count > 0)
{
var objectsDeleteRequest = new DeleteObjectsRequest()
{
BucketName = _bucket,
Objects = keys
};
await _s3Client.DeleteObjectsAsync(objectsDeleteRequest);
}
}
catch (AmazonS3Exception asex)
{
throw asex.ToStorageException();
}
}
public void S3DeleteItemsFolder(string bucketName, string serverFolder)
{
//ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingNetSDK.html
int count = 0;
using (var client = new AmazonS3Client(this.AcesssKey, this.SecretKey, this.Region))
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName,
MaxKeys = 10,
Prefix = serverFolder
};
do
{
ListObjectsResponse response = client.ListObjects(request);
// Process response
foreach (S3Object entry in response.S3Objects)
{
if (entry.Key == serverFolder || entry.Key == string.Format("{0}/", serverFolder) || entry.Key == string.Format("/{0}", serverFolder))
continue; //Folder
count++;
//System.Diagnostics.Debug.WriteLine(string.Format("AwsS3 -- key = {0} size = {1} / {2} items read", entry.Key, entry.Size.ToString("#,##0"), count.ToString("#,##0")));
this.S3DeleteItem(bucketName, entry.Key);
Console.WriteLine(string.Format("{0} -- {1} items deleted", entry.Key, count.ToString("#,##0")));
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
}
}
public static void DeleteAllBucketItems(string bucketName)
{
using (var client = new AmazonS3Client(Settings.AccessKey, Settings.Secret))
{
var request = new ListObjectsRequest
{
BucketName = bucketName
};
var response = client.ListObjects(request);
foreach (var entry in response.S3Objects)
{
client.DeleteObject(bucketName, entry.Key);
}
}
}
public IEnumerable<FileData> GetFiles(string parentVirtualPath) {
var request = new ListObjectsRequest()
.WithBucketName(this.bucketName)
.WithPrefix(FixPathForS3(parentVirtualPath))
.WithDelimiter(@"/");
IEnumerable<FileData> files;
using (var response = this.s3.ListObjects(request)) {
files = response.S3Objects.Where(file => file.Size > 0).Select(file => new FileData {
VirtualPath = FixPathForN2(file.Key),
Name = file.Key.Substring(file.Key.LastIndexOf('/') + 1),
Length = file.Size,
Created = DateTime.Parse(file.LastModified),
Updated = DateTime.Parse(file.LastModified)
});
}
return files;
}
public void csv(string bucket)
{
string nextmarker = "";
try
{
Console.Write("\"BUCKET_NAME\"");
Console.Write(",");
Console.Write("\"KEY_NAME\"");
Console.Write(",");
Console.Write("\"LAST_MODIFIED\"");
Console.Write(",");
Console.WriteLine("\"SIZE_BYTES\"");
while (nextmarker != null)
{
ListObjectsRequest listrequest = new ListObjectsRequest();
listrequest.BucketName = bucket;
listrequest.Marker = nextmarker;
ListObjectsResponse listresponse = s3Client.ListObjects(listrequest);
foreach (S3Object s3obj in listresponse.S3Objects)
{
Console.Write("\"" + bucket + "\"");
Console.Write(",");
Console.Write("\"" + s3obj.Key + "\"");
Console.Write(",");
Console.Write("\"" + s3obj.LastModified.ToString() + "\"");
Console.Write(",");
Console.WriteLine(s3obj.Size);
}
if (listresponse.S3Objects.Count == 1000)
{
nextmarker = listresponse.S3Objects[999].Key;
}
else
{
nextmarker = null;
}
}
}
catch (Exception e)
{
// report error and retry
Console.WriteLine("Error retrieving bucket list for bucket '" + bucket + "': " + e.Message);
}
}
/// <summary>
/// Deletes all keys in a given bucket, then deletes the bucket itself.
/// </summary>
/// <param name="client">The client to use.</param>
/// <param name="bucketName">The bucket to delete.</param>
public static void DeleteBucketRecursive(this AmazonS3 client, string bucketName)
{
while(true)
{
// attempt to delete the bucket
try
{
var deleteRequest = new DeleteBucketRequest()
{
BucketName = bucketName
};
using (var deleteResponse = client.DeleteBucket(deleteRequest))
{
// deletion was successful
return;
}
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode != "BucketNotEmpty") throw ex;
}
var objRequest = new ListObjectsRequest()
{
BucketName = bucketName
};
using (var objResponse = client.ListObjects(objRequest))
{
var delRequest = new DeleteObjectsRequest()
{
BucketName = bucketName,
Quiet = true
};
// select the objects to delete (up to the supported limit of 1000)
var objToDelete = objResponse.S3Objects.Take(1000).Select(o => new KeyVersion(o.Key));
delRequest.AddKeys(objToDelete.ToArray());
using (var delResponse = client.DeleteObjects(delRequest))
{
}
}
}
}
public void ProcessS3Objects(string prefix, ProcessS3ObjectDelegate processS3ObjectDelegate)
{
ListObjectsRequest request = new ListObjectsRequest {
BucketName = mBucketName,
Prefix = prefix
};
bool proceed = true;
do
{
ListObjectsResponse response = mS3Client.ListObjects(request);
foreach (S3Object obj in response.S3Objects)
{
if (!(proceed = processS3ObjectDelegate(obj))) { break; }
}
if (proceed && response.IsTruncated) { request.Marker = response.NextMarker; }
else { proceed = false; }
}
while (proceed);
}
public ActionResult Index()
{
// get me all objects inside a given folder
Dictionary<string, double> images = null;
var request = new ListObjectsRequest();
request.BucketName = AWSBucket;
request.WithPrefix(AWSFolder);
using (AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(AWSAccessKey, AWSSecretKey))
{
using (ListObjectsResponse response = client.ListObjects(request))
{
images = response.S3Objects.Where(x => x.Key != AWSFolder).ToDictionary(obj => obj.Key, obj => AppHelper.ConvertBytesToMegabytes(obj.Size));
}
}
return View(images);
}
/// <summary>
/// 設定されたバケットからカレントディレクトリにすべてのファイルを同期する。
/// </summary>
/// <param name="timeFilter"></param>
public override void Run()
{
// トリガーモードで、ローカルのトリガーが新しいなら何もしない
if (triggerState == TriggerState.Local || triggerState == TriggerState.Same)
{
Console.WriteLine("Trigger is not updated. abort execution.");
return;
}
CopiedFiles.Clear();
if (string.IsNullOrEmpty(Bucket))
throw new Exception("バケット名が指定されていません。");
ListObjectsResponse response = new ListObjectsResponse();
do
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = Bucket,
MaxKeys = int.MaxValue,
Marker = response.NextMarker,
};
response = transferUtility.S3Client.ListObjects(request);
// response.
foreach (var o in response.S3Objects)
{
var filename = procKey(o.Key);
filename = Path.Combine(TargetDirectory, filename);
var dirname = Path.GetDirectoryName(filename);
if (!Directory.Exists(dirname))
{
Directory.CreateDirectory(dirname);
}
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
retrieveFile(o, filename);
}
}
} while (!string.IsNullOrEmpty(response.NextMarker));
}
public IEnumerable<DirectoryData> GetDirectories(string parentVirtualPath) {
parentVirtualPath = FixPathForS3(parentVirtualPath);
var request = new ListObjectsRequest()
.WithBucketName(this.bucketName)
.WithPrefix(parentVirtualPath)
.WithDelimiter(@"/");
IEnumerable<DirectoryData> directories;
using (var response = this.s3.ListObjects(request)) {
directories = response.CommonPrefixes.Select(dir => new DirectoryData {
Created = DateTime.Now,
Updated = DateTime.Now,
Name = dir.TrimEnd('/').Substring(dir.TrimEnd('/').LastIndexOf('/') + 1),
VirtualPath = FixPathForN2(dir)
});
}
return directories;
}
/// <summary>
/// Delete a folder
/// </summary>
/// <param name="prefix">prefix</param>
public void DeleteFolder(string prefix)
{
// Get all object with specified prefix
var listRequest = new ListObjectsRequest()
{
BucketName = _bucketName,
Prefix = prefix
};
var deleteRequest = new DeleteObjectsRequest
{
BucketName = _bucketName
};
do
{
var listResponse = _client.ListObjects(listRequest);
// Add all object with specified prefix to delete request.
foreach (var entry in listResponse.S3Objects)
{
deleteRequest.AddKey(entry.Key);
}
if (listResponse.IsTruncated)
{
listRequest.Marker = listResponse.NextMarker;
}
else
{
listRequest = null;
}
}
while (listRequest != null);
// Delete all the object with specified prefix.
if (deleteRequest.Objects.Count > 0)
{
var deleteResponse = _client.DeleteObjects(deleteRequest);
deleteResponse.DisposeIfDisposable();
}
}
private void DisplayImageList()
{
ListBucketsResponse response = S3Client.ListBuckets();
foreach (S3Bucket bucket in response.Buckets)
{
HtmlGenericControl header = new HtmlGenericControl("div")
{
InnerText = bucket.BucketName
};
DivlistOfImages.Controls.Add(header);
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucket.BucketName
};
ListObjectsResponse imageList = S3Client.ListObjects(request);
foreach (S3Object s3Object in imageList.S3Objects)
{
HtmlAnchor imageLink = new HtmlAnchor
{
InnerText = s3Object.Key
};
string bucketName = bucket.BucketName;
string objectKey = s3Object.Key;
GetPreSignedUrlRequest preSignedUrlRequest = new GetPreSignedUrlRequest
{
BucketName = bucketName,
Key = objectKey,
Protocol = Protocol.HTTP,
Expires = DateTime.Now.AddDays(7)
};
imageLink.HRef = S3Client.GetPreSignedURL(preSignedUrlRequest);
DivlistOfImages.Controls.Add(imageLink);
}
}
}
async Task<IList<string>> ICoreAmazonS3.GetAllObjectKeysAsync(string bucketName, string prefix, IDictionary<string, object> additionalProperties)
{
var keys = new List<string>();
string marker = null;
do
{
var request = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = prefix,
Marker = marker
};
InternalSDKUtils.ApplyValues(request, additionalProperties);
var listResponse = await this.ListObjectsAsync(request).ConfigureAwait(false);
keys.AddRange(listResponse.S3Objects.Select(o => o.Key));
marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(marker));
return keys;
}
public void Dispose()
{
var objectsRequest = new ListObjectsRequest
{
BucketName = Config["Bucket"],
Prefix = ContainerPrefix,
MaxKeys = 100000
};
var keys = new List<KeyVersion>();
do
{
var objectsResponse = _client.ListObjectsAsync(objectsRequest).Result;
keys.AddRange(objectsResponse.S3Objects
.Select(x => new KeyVersion() { Key = x.Key, VersionId = null }));
// If response is truncated, set the marker to get the next set of keys.
if (objectsResponse.IsTruncated)
{
objectsRequest.Marker = objectsResponse.NextMarker;
}
else
{
objectsRequest = null;
}
} while (objectsRequest != null);
if (keys.Count > 0)
{
var objectsDeleteRequest = new DeleteObjectsRequest()
{
BucketName = Config["Bucket"],
Objects = keys
};
_client.DeleteObjectsAsync(objectsDeleteRequest).Wait();
}
}
/// <summary>
/// S3バケットのみに含まれるファイルを検出
/// </summary>
/// <returns></returns>
public List<string> FindDeletedFiles()
{
var ret = new List<string>();
CopiedFiles.Clear();
if (string.IsNullOrEmpty(Bucket))
throw new Exception("バケット名が指定されていません。");
ListObjectsResponse response = new ListObjectsResponse();
do
{
// S3バケット内のオブジェクト取得リクエストを構築
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = Bucket,
MaxKeys = int.MaxValue, // 取れるだけを要求するが、実際に取れるのは1000件
Marker = response.NextMarker, // 取得開始カーソル
};
response = transferUtility.S3Client.ListObjects(request);
foreach (var o in response.S3Objects)
{
// S3のキー名から対応するローカルファイル名を生成
var filename = Path.Combine(TargetDirectory,procKey(o.Key));
var dirname = Path.GetDirectoryName(filename);
if (!Directory.Exists(dirname) || // 存在しないディレクトリ
string.IsNullOrEmpty(Path.GetFileName(filename)) || // ファイル名が無い
!File.Exists(filename)) // 存在しない
{
ret.Add(o.Key);
}
}
} while (!string.IsNullOrEmpty(response.NextMarker));
return ret;
}
private Amazon.S3.Model.ListObjectsResponse CallAWSServiceOperation(IAmazonS3 client, Amazon.S3.Model.ListObjectsRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Simple Storage Service (S3)", "ListObjects");
try
{
#if DESKTOP
return(client.ListObjects(request));
#elif CORECLR
return(client.ListObjectsAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
var useParameterSelect = this.Select.StartsWith("^") || this.PassThru.IsPresent;
// create request and set iteration invariants
var request = new Amazon.S3.Model.ListObjectsRequest();
if (cmdletContext.BucketName != null)
{
request.BucketName = cmdletContext.BucketName;
}
if (cmdletContext.Delimiter != null)
{
request.Delimiter = cmdletContext.Delimiter;
}
if (cmdletContext.Prefix != null)
{
request.Prefix = cmdletContext.Prefix;
}
if (cmdletContext.Encoding != null)
{
request.Encoding = cmdletContext.Encoding;
}
if (cmdletContext.RequestPayer != null)
{
request.RequestPayer = cmdletContext.RequestPayer;
}
// Initialize loop variants and commence piping
System.String _nextToken = null;
int? _emitLimit = null;
int _retrievedSoFar = 0;
if (AutoIterationHelpers.HasValue(cmdletContext.Marker))
{
_nextToken = cmdletContext.Marker;
}
if (cmdletContext.MaxKey.HasValue)
{
// The service has a maximum page size of 1000. If the user has
// asked for more items than page max, and there is no page size
// configured, we rely on the service ignoring the set maximum
// and giving us 1000 items back. If a page size is set, that will
// be used to configure the pagination.
// We'll make further calls to satisfy the user's request.
_emitLimit = cmdletContext.MaxKey;
}
var _userControllingPaging = this.NoAutoIteration.IsPresent || ParameterWasBound(nameof(this.Marker));
var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint);
do
{
request.Marker = _nextToken;
if (_emitLimit.HasValue)
{
int correctPageSize = Math.Min(1000, _emitLimit.Value);
request.MaxKeys = AutoIterationHelpers.ConvertEmitLimitToInt32(correctPageSize);
}
CmdletOutput output;
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = null;
if (!useParameterSelect)
{
pipelineOutput = cmdletContext.Select(response, this);
}
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
int _receivedThisCall = response.S3Objects.Count;
_nextToken = response.NextMarker;
_retrievedSoFar += _receivedThisCall;
if (_emitLimit.HasValue)
{
_emitLimit -= _receivedThisCall;
}
}
catch (Exception e)
{
if (_retrievedSoFar == 0 || !_emitLimit.HasValue)
{
output = new CmdletOutput {
ErrorResponse = e
};
}
else
{
break;
}
}
ProcessOutput(output);
} while (!_userControllingPaging && AutoIterationHelpers.HasValue(_nextToken) && (!_emitLimit.HasValue || _emitLimit.Value >= 1));
if (useParameterSelect)
{
WriteObject(cmdletContext.Select(null, this));
}
return(null);
}
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
var useParameterSelect = this.Select.StartsWith("^") || this.PassThru.IsPresent;
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute
// create request and set iteration invariants
var request = new Amazon.S3.Model.ListObjectsRequest();
if (cmdletContext.BucketName != null)
{
request.BucketName = cmdletContext.BucketName;
}
if (cmdletContext.Delimiter != null)
{
request.Delimiter = cmdletContext.Delimiter;
}
if (cmdletContext.MaxKey != null)
{
request.MaxKeys = AutoIterationHelpers.ConvertEmitLimitToServiceTypeInt32(cmdletContext.MaxKey.Value);
}
if (cmdletContext.Prefix != null)
{
request.Prefix = cmdletContext.Prefix;
}
if (cmdletContext.Encoding != null)
{
request.Encoding = cmdletContext.Encoding;
}
if (cmdletContext.RequestPayer != null)
{
request.RequestPayer = cmdletContext.RequestPayer;
}
// Initialize loop variant and commence piping
var _nextToken = cmdletContext.Marker;
var _userControllingPaging = this.NoAutoIteration.IsPresent || ParameterWasBound(nameof(this.Marker));
var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint);
do
{
request.Marker = _nextToken;
CmdletOutput output;
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = null;
if (!useParameterSelect)
{
pipelineOutput = cmdletContext.Select(response, this);
}
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
_nextToken = response.NextMarker;
}
catch (Exception e)
{
output = new CmdletOutput {
ErrorResponse = e
};
}
ProcessOutput(output);
} while (!_userControllingPaging && AutoIterationHelpers.HasValue(_nextToken));
if (useParameterSelect)
{
WriteObject(cmdletContext.Select(null, this));
}
return(null);
}
internal ListObjectsPaginator(IAmazonS3 client, ListObjectsRequest request)
{
this._client = client;
this._request = request;
}
private void rbnButton_Click(object sender, EventArgs e)
{
RadioButton myRadioButton = (RadioButton)sender;
if (myRadioButton.Name == "rbnLocal")
{
this.btnOK.Text = "Restore";
Get_Local_Backup_Files();
}
else
{
//Remote
this.btnOK.Text = "Download";
this.dgvFilesDataGridView.Rows.Clear();
this.Refresh();
string strCompanyNo = "";
DateTime dtDateTime = DateTime.Now;
pvtLocalBackupFilesDataView = null;
pvtLocalBackupFilesDataView = new DataView(pvtDataSet.Tables["LocalBackupFiles"],
"",
"LOCAL_FILE",
DataViewRowState.CurrentRows);
ListObjectsRequest listObjectsRequest = new Amazon.S3.Model.ListObjectsRequest();
listObjectsRequest.BucketName = pvtstrBucketName;
listObjectsRequest.Prefix = "InteractPayroll";
ListObjectsResponse listObjectsResponse = iAmazonS3.ListObjects(listObjectsRequest);
//Reverse Order of Downloaded Files
listObjectsResponse.S3Objects.Reverse();
foreach (S3Object s3Object in listObjectsResponse.S3Objects)
{
int intFindRow = pvtLocalBackupFilesDataView.Find(s3Object.Key);
if (intFindRow > -1)
{
continue;
}
strCompanyNo = "";
try
{
if (s3Object.Key.Substring(15, 1) == "_" &&
s3Object.Key.Substring(21, 1) == "_")
{
strCompanyNo = s3Object.Key.Substring(16, 5);
string strdate = s3Object.Key.Substring(22, 8) + s3Object.Key.Substring(31, 6);
dtDateTime = DateTime.ParseExact(strdate, "yyyyMMddHHmmss", null);
}
else
{
string strdate = s3Object.Key.Substring(16, 8) + s3Object.Key.Substring(25, 6);
dtDateTime = DateTime.ParseExact(strdate, "yyyyMMddHHmmss", null);
}
this.dgvFilesDataGridView.Rows.Add(dtDateTime.ToString("yyyy-MM-dd HH:mm:ss"),
strCompanyNo,
s3Object.Key);
}
catch
{
//Backup Not From Program
}
}
}
}
/// <summary>
/// Paginator for ListObjects operation
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public IListObjectsPaginator ListObjects(ListObjectsRequest request)
{
return(new ListObjectsPaginator(this._client, request));
}
