private async Task LoadImages()
{
var client = S3Utils.S3Client;
response = await client.ListObjectsAsync(new ListObjectsRequest()
{
BucketName = Constants.BUCKET_NAME.ToLowerInvariant(),
Prefix = "myPhoto",
MaxKeys = 20
});
response.S3Objects.ForEach(async (s3Object) =>
{
var res = await client.GetObjectAsync(new GetObjectRequest()
{
BucketName = Constants.BUCKET_NAME.ToLowerInvariant(),
Key = s3Object.Key
});
using (var stream = res.ResponseStream)
{
var b = ReadFully(stream);
var bitmap = BitmapFactory.DecodeByteArray(b, 0, b.Length);
images.Add(bitmap);
}
RunOnUiThread(() =>
{
var adp = gridView.Adapter as ImageAdapter;
adp.NotifyDataSetChanged();
});
});
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
ListObjectsResponse response = new ListObjectsResponse();
while (context.Read())
{
if (context.IsStartElement)
{
UnmarshallResult(context,response);
continue;
}
}
return response;
}
/// <summary>
/// 設定されたバケットからカレントディレクトリにすべてのファイルを同期する。
/// </summary>
/// <param name="timeFilter"></param>
public override void Run()
{
// トリガーモードで、ローカルのトリガーが新しいなら何もしない
if (triggerState == TriggerState.Local || triggerState == TriggerState.Same)
{
Console.WriteLine("Trigger is not updated. abort execution.");
return;
}
CopiedFiles.Clear();
if (string.IsNullOrEmpty(Bucket))
throw new Exception("バケット名が指定されていません。");
ListObjectsResponse response = new ListObjectsResponse();
do
{
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = Bucket,
MaxKeys = int.MaxValue,
Marker = response.NextMarker,
};
response = transferUtility.S3Client.ListObjects(request);
// response.
foreach (var o in response.S3Objects)
{
var filename = procKey(o.Key);
filename = Path.Combine(TargetDirectory, filename);
var dirname = Path.GetDirectoryName(filename);
if (!Directory.Exists(dirname))
{
Directory.CreateDirectory(dirname);
}
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
retrieveFile(o, filename);
}
}
} while (!string.IsNullOrEmpty(response.NextMarker));
}
/// <summary>
/// S3バケットのみに含まれるファイルを検出
/// </summary>
/// <returns></returns>
public List<string> FindDeletedFiles()
{
var ret = new List<string>();
CopiedFiles.Clear();
if (string.IsNullOrEmpty(Bucket))
throw new Exception("バケット名が指定されていません。");
ListObjectsResponse response = new ListObjectsResponse();
do
{
// S3バケット内のオブジェクト取得リクエストを構築
ListObjectsRequest request = new ListObjectsRequest
{
BucketName = Bucket,
MaxKeys = int.MaxValue, // 取れるだけを要求するが、実際に取れるのは1000件
Marker = response.NextMarker, // 取得開始カーソル
};
response = transferUtility.S3Client.ListObjects(request);
foreach (var o in response.S3Objects)
{
// S3のキー名から対応するローカルファイル名を生成
var filename = Path.Combine(TargetDirectory,procKey(o.Key));
var dirname = Path.GetDirectoryName(filename);
if (!Directory.Exists(dirname) || // 存在しないディレクトリ
string.IsNullOrEmpty(Path.GetFileName(filename)) || // ファイル名が無い
!File.Exists(filename)) // 存在しない
{
ret.Add(o.Key);
}
}
} while (!string.IsNullOrEmpty(response.NextMarker));
return ret;
}
public void DeleteDirectory()
{
//Arrange
var response = new ListObjectsResponse { IsTruncated = false };
response.S3Objects.AddRange(new [] {
new S3Object { Key = "media/abc/object1" },
new S3Object { Key = "media/abc/object2" }
});
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/abc")))
.Returns(response);
clientMock.Setup(p => p.DeleteObjects(It.IsAny<DeleteObjectsRequest>()))
.Returns(new DeleteObjectsResponse { DeletedObjects = new List<DeletedObject>() });
var provider = CreateProvider(clientMock);
//Act
provider.DeleteDirectory("/abc");
//Assert
clientMock.VerifyAll();
}
private static void UnmarshallResult(XmlUnmarshallerContext context,ListObjectsResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
if (context.IsStartOfDocument)
targetDepth += 2;
while (context.Read())
{
if (context.IsStartElement || context.IsAttribute)
{
if (context.TestExpression("IsTruncated", targetDepth))
{
response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("NextMarker", targetDepth))
{
response.NextMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("Contents", targetDepth))
{
response.S3Objects.Add(ContentsItemUnmarshaller.Instance.Unmarshall(context));
continue;
}
if (context.TestExpression("Name", targetDepth))
{
response.Name = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("Prefix", targetDepth))
{
response.Prefix = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("Delimiter", targetDepth))
{
response.Delimiter = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("MaxKeys", targetDepth))
{
response.MaxKeys = IntUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("CommonPrefixes", targetDepth))
{
var prefix = CommonPrefixesItemUnmarshaller.Instance.Unmarshall(context);
if(prefix != null)
response.CommonPrefixes.Add(prefix);
continue;
}
}
else if (context.IsEndElement && context.CurrentDepth < originalDepth)
{
return;
}
}
return;
}
private int CountDesiredFiles(ListObjectsResponse matching)
{
int totalItems = 0;
for (int i = 0; i < matching.S3Objects.Count; ++i)
{
if (AvoidThisFile(matching.S3Objects[i].Key))
continue;
++totalItems;
}
return totalItems;
}
public void Given_an_s3_bucket_with_some_files_and_folders()
{
List<S3Object> fakeObjects = new List<S3Object>()
{
new S3Object()
{
Key = "subfolder/File.bin",
Size = 100,
LastModified = AWSSDKUtils.FormattedCurrentTimestampISO8601,
BucketName = "BucketName"
},
new S3Object()
{
Key = "subfolder2/subfolder3/",
LastModified = AWSSDKUtils.FormattedCurrentTimestampISO8601,
BucketName = "BucketName"
}
};
var fakeResponse = new ListObjectsResponse();
fakeResponse.S3Objects.AddRange(fakeObjects);
MockS3.Setup(p => p.ListObjects(
It.Is<ListObjectsRequest>(req => req.BucketName == "BucketName")
)).Returns(fakeResponse);
}
private IEnumerable<RemoteResource> MapResponse(ListObjectsResponse listResponse)
{
var resources = new List<RemoteResource>();
var objects = listResponse.S3Objects;
resources.AddRange(objects.Where(e => !e.Key.EndsWith("/"))
.Select(e => new RemoteResource()
{
Host = _bucket,
Name = GetObjectName(e.Key),
Path = e.Key,
ResourceType = ResourceType.File,
Size = e.Size,
StorageType = StorageType.S3
}));
resources.AddRange(listResponse.CommonPrefixes
.Select(f => new RemoteResource()
{
Host = _bucket,
Name = GetObjectName(f),
Path = f,
ResourceType = ResourceType.Directory,
StorageType = StorageType.S3
}));
return resources;
}
public void DeleteDirectoryWithContinuationMarker()
{
//Arrange
var response1 = new ListObjectsResponse { IsTruncated = true, NextMarker = "Marker1" };
response1.S3Objects.AddRange(new [] {
new S3Object { Key = "media/object1" },
new S3Object { Key = "media/object2" }
});
var response2 = new ListObjectsResponse { IsTruncated = false };
response2.S3Objects.AddRange(new [] {
new S3Object { Key = "media/object3" },
new S3Object { Key = "media/object4" }
});
var response3 = new DeleteObjectsResponse();
response3.DeletedObjects = new List<DeletedObject>(new[] {
new DeletedObject { DeleteMarker = true, Key = "media/object1" },
new DeletedObject { DeleteMarker = true, Key = "media/object2" },
new DeletedObject { DeleteMarker = true, Key = "media/object3" },
new DeletedObject { DeleteMarker = true, Key = "media/object4" }
});
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/" && req.Marker == null)))
.Returns(response1);
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/" && req.Marker == "Marker1")))
.Returns(response2);
clientMock.Setup(p => p.DeleteObjects(It.IsAny<DeleteObjectsRequest>()))
.Returns(response3);
var provider = CreateProvider(clientMock);
//Act
provider.DeleteDirectory("/");
//Assert
clientMock.VerifyAll();
}
public void GetFilesWithContinuationMarker()
{
//Arrange
var response1 = new ListObjectsResponse { IsTruncated = true, NextMarker = "marker1" };
response1.S3Objects.AddRange(new [] {
new S3Object { Key = "media/object1" },
new S3Object { Key = "media/object2" }
});
var response2 = new ListObjectsResponse { IsTruncated = false };
response2.S3Objects.AddRange(new [] {
new S3Object { Key = "media/object3" },
new S3Object { Key = "media/object4" }
});
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/1001/" && req.Delimiter == "/" && req.Marker == null)))
.Returns(response1);
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/1001/" && req.Delimiter == "/" && req.Marker == "marker1")))
.Returns(response2);
var provider = CreateProvider(clientMock);
//Act
var actual = provider.GetFiles("1001/");
//Assert
var expected = new[] { "object1", "object2", "object3", "object4" };
Assert.IsTrue(expected.SequenceEqual(actual));
clientMock.VerifyAll();
}
public void GetDirectoriesWithContinuationMarker()
{
//Arrange
var response1 = new ListObjectsResponse { IsTruncated = true, NextMarker = "Marker1" };
response1.CommonPrefixes.AddRange(new[] { "media/1001/", "media/1002/", "media/1003/" });
var response2 = new ListObjectsResponse { IsTruncated = false };
response2.CommonPrefixes.AddRange(new[] { "media/1004/", "media/1005/", "media/1006/" });
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/" && req.Delimiter == "/" && req.Marker == null)))
.Returns(response1);
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Prefix == "media/" && req.Delimiter == "/" && req.Marker == "Marker1")))
.Returns(response2);
var provider = CreateProvider(clientMock);
//Act
var actual = provider.GetDirectories("/");
//Assert
var expected = new[] { "1001", "1002", "1003", "1004", "1005", "1006" };
Assert.IsTrue(expected.SequenceEqual(actual));
clientMock.VerifyAll();
}
public void GetDirectories()
{
//Arrange
var response = new ListObjectsResponse { IsTruncated = false };
response.CommonPrefixes.AddRange(new[] { "media/1010/", "media/1011/", "media/1012/" });
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.Delimiter == "/" && req.Prefix == "media/")))
.Returns(response);
var provider = CreateProvider(clientMock);
//Act
var actual = provider.GetDirectories("/");
//Assert
var expected = new[] {"1010", "1011", "1012"};
Assert.IsTrue(expected.SequenceEqual(actual));
clientMock.VerifyAll();
}
public void DirectoryExistsNoObjects()
{
//Arrange
var listResponse = new ListObjectsResponse { IsTruncated = false };
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.MaxKeys == 1)))
.Returns(listResponse);
var provider = CreateProvider(clientMock);
//Act
var actual = provider.DirectoryExists("/abc");
//Arrange
Assert.IsFalse(actual);
clientMock.VerifyAll();
}
public void DirectoryExists()
{
//Arrange
var response = new ListObjectsResponse { IsTruncated = false };
response.S3Objects.AddRange(new [] {
new S3Object { Key = "abc/object1" },
new S3Object { Key = "abc/object2" }
});
var clientMock = new Mock<IAmazonS3>();
clientMock.Setup(p => p.ListObjects(It.Is<ListObjectsRequest>(req => req.MaxKeys == 1)))
.Returns(response);
var provider = CreateProvider(clientMock);
//Act
var actual = provider.DirectoryExists("/abc");
//Arrange
Assert.IsTrue(actual);
clientMock.VerifyAll();
}