public List <object> GetFileObjects()
{
AmazonS3 s3 = AWSClientFactory.CreateAmazonS3Client();
List <S3Object> Files = new List <S3Object>();
string nextMarker = "";
do
{
ListObjectsRequest lor = new ListObjectsRequest().WithBucketName(AmazonBucket).WithPrefix(AmazonPrefix).WithMarker(nextMarker);
var response = s3.ListObjects(lor);
if (response.IsTruncated)
{
nextMarker = response.NextMarker;
}
else
{
nextMarker = "";
}
Files.AddRange(response.S3Objects);
System.Diagnostics.Trace.WriteLine(string.Format("Added {0} files. Total files: {1}", response.S3Objects.Count, Files.Count));
} while (nextMarker != "");
return(Files.Cast <object>().ToList());
//var buckets = ComputeNode.Catalogs.Values.Cast<ICatalog>().Where(c => c.CatalogName == "WikipediaData").First().Buckets;
//var bucketMods = buckets.Select(b => b.Value.BucketMod).ToList();
//myFiles = allFiles.Where(f => bucketMods.Contains(f.GetHashCode() % ComputeNode.GlobalBucketCount)).OrderBy(f => f.Key).ToList();
}
private void GetFileListing()
{
s3 = AWSClientFactory.CreateAmazonS3Client();
string nextMarker = "";
do
{
ListObjectsRequest lor = new ListObjectsRequest().WithBucketName(BUCKET).WithPrefix(PREFIX).WithMarker(nextMarker);
var response = s3.ListObjects(lor);
if (response.IsTruncated)
{
nextMarker = response.NextMarker;
}
else
{
nextMarker = "";
}
allFiles.AddRange(response.S3Objects);
Trace.WriteLine(string.Format("Added {0} files. Total files: {1}", response.S3Objects.Count, allFiles.Count));
} while (nextMarker != "");
var buckets = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName == "WikipediaData").First().Buckets;
var bucketMods = buckets.Select(b => b.Value.BucketMod).ToList();
myFiles = allFiles.Where(f => bucketMods.Contains(f.GetHashCode() % ComputeNode.GlobalBucketCount)).OrderBy(f => f.Key).ToList();
}
static void Main(string[] args)
{
foreach (var enumerableAllRegion in RegionEndpoint.EnumerableAllRegions)
{
System.Console.Out.WriteLine(enumerableAllRegion.DisplayName + " " + enumerableAllRegion.SystemName);
}
System.Net.WebRequest.DefaultWebProxy = WebRequest.GetSystemWebProxy();
AmazonS3Config config = new AmazonS3Config();
//config.ProxyHost = "localhost";
//config.ProxyPort = 3128;
//config.ProxyCredentials = new
config.RegionEndpoint = RegionEndpoint.EUWest1;
//config.UseHttp = true;
var client = AWSClientFactory.CreateAmazonS3Client("", "", config);
//client.
var response = client.ListBuckets();
foreach (var bucket in response.Buckets)
{
System.Console.Out.WriteLine(bucket.BucketName);
}
System.Console.Out.WriteLine("--end--");
System.Console.ReadLine();
}
public static async Task <OperationResult <string> > DeleteFromProvider(string path)
{
try
{
IAmazonS3 client;
using (client = AWSClientFactory.CreateAmazonS3Client(_awsAccessKey, _awsSecretKey, RegionEndpoint.GetBySystemName("eu-west-1")))
{
var request = new DeleteObjectRequest()
{
BucketName = _bucketName,
Key = path
};
await client.DeleteObjectAsync(request);
return(new OperationResult <string> {
Success = true, Result = image_start_path
});
}
}
catch (Exception ex)
{
return(new OperationResult <string> {
Success = false, Result = image_default, Message = ex.Message
});
}
}
public FileRepository()
{
_config = ConfigurationManager.GetSection("AspGuy/S3Repository") as S3FileRepositoryConfig;
if (_config == null)
{
throw new ConfigurationErrorsException(
"A configuration section for S3FileRepositoryConfig is expected but it is missing in the configuration file.");
}
bool aConfigValueIsMissing = string.IsNullOrEmpty(_config.AccessKey) ||
string.IsNullOrEmpty(_config.SecretKey) ||
string.IsNullOrEmpty(_config.RegionName) ||
string.IsNullOrEmpty(_config.RootBucketName);
if (aConfigValueIsMissing)
{
throw new ConfigurationErrorsException(
"A configuration attribute of S3FileRepositoryConfig class is missing. Please check the configuration file.");
}
_client = AWSClientFactory.CreateAmazonS3Client(_config.AccessKey, _config.SecretKey,
RegionEndpoint.GetBySystemName(_config.RegionName));
_transferAgent = new TransferUtility(_client);
_workingDir = _config.RootDir;
}
public void SetUp()
{
AWSCredentials credentials = AmbientCredentials.GetCredentials();
S3Client = AWSClientFactory.CreateAmazonS3Client(credentials);
StorageService = new StorageService(S3Client, new S3PathParser());
}
private static Boolean MoveFile(string saveAsFileName, string wkBaseFolder, Int32 wkVersionNo, string currentFilePath)
{
try
{
string destFile = wkBaseFolder + "/" + wkVersionNo.ToString() + "/" + saveAsFileName;
AmazonS3Config config = new AmazonS3Config();
config.ServiceURL = AWSHelpers.GetS3EndPoint();
using (var client = AWSClientFactory.CreateAmazonS3Client(AWSHelpers.GetAccesskey(), AWSHelpers.GetSecretkey(), config))
{
TransferUtility s3TransferUtil = new TransferUtility(client);
TransferUtilityUploadRequest s3TrfrReq = new TransferUtilityUploadRequest();
s3TrfrReq.CannedACL = Amazon.S3.Model.S3CannedACL.BucketOwnerFullControl;
s3TrfrReq.FilePath = currentFilePath;
s3TrfrReq.BucketName = AWSHelpers.GetBucketname();
s3TrfrReq.Key = destFile;
s3TransferUtil.Upload(s3TrfrReq);
// Delete the temporary PDF on the web server
FileInfo TheFile = new FileInfo(s3TrfrReq.FilePath);
if (TheFile.Exists)
{
// File found so delete it.
TheFile.Delete();
}
}
return(true);
}
catch (Exception ex)
{
return(false);
}
}
private static IAmazonS3 GetS3Client(string choice)
{
if (string.IsNullOrWhiteSpace(choice) == false && choice.Trim().ToLower() == "y")
{
try
{
// here we use the IAM Role attached to the EC2 instance to get the credentials
return(new AmazonS3Client(RegionEndpoint.USWest2));
}
catch (Exception ex)
{
WriteLine(ConsoleColor.Red, ex.Message);
return(null);
}
}
var config = new AmazonS3Config {
RegionEndpoint = RegionEndpoint.USWest2
};
var client = AWSClientFactory.CreateAmazonS3Client(
AccessKey,
SecretKey,
config
);
return(client);
}
public S3FileSystem()
{
this.accessKeyId = ConfigurationManager.AppSettings["AWSAccessKeyID"];
this.secretAccessKey = ConfigurationManager.AppSettings["AWSSecretAccessKey"];
this.bucketName = ConfigurationManager.AppSettings["AWSBucketName"];
this.s3 = AWSClientFactory.CreateAmazonS3Client(this.accessKeyId, this.secretAccessKey);
}
public static async Task <bool> TryPutAsync(string bucket, string key, Stream stream)
{
var length = stream.Length;
// no need to check for existence because S3 will overwrite
// files with the same key -> no more space automatically
// and save one request without cheking for existence
// could be performance issue for large files
const int checkExistenceLimit = 1 * 1024 * 1024; // 1 Mb
if (length > checkExistenceLimit)
{
if (Exists(bucket, key))
{
return(true);
}
}
using (IAmazonS3 client = AWSClientFactory.CreateAmazonS3Client(_endpoint)) {
try {
var request = new PutObjectRequest {
BucketName = bucket,
Key = key,
InputStream = stream
};
// ReSharper disable once UnusedVariable
var response = await client.PutObjectAsync(request);
return(true);
} catch {
return(false);
}
}
}
public byte[] FetchFile(string sObjectKey, string sVersionId)
{
AmazonS3 client = AWSClientFactory.CreateAmazonS3Client(S3ACCESSKEY, S3SECRETKEY);
string BUCKET_NAME = ConfigurationManager.AppSettings["AWSBUCKET"];
GetObjectRequest request = new GetObjectRequest();
request.WithKey(sObjectKey);
request.WithBucketName(BUCKET_NAME);
if (sVersionId != "")
{
request.WithVersionId(sVersionId);
}
GetObjectResponse response = client.GetObject(request);
byte[] buffer = new byte[response.ContentLength];
int read;
MemoryStream ms = new MemoryStream();
while ((read = response.ResponseStream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
return(ms.ToArray());
}
public ListVersionsResponse MssListFileVersions(string sObjectKey)
{
AmazonS3 client = AWSClientFactory.CreateAmazonS3Client(S3ACCESSKEY, S3SECRETKEY);
string BUCKET_NAME = ConfigurationManager.AppSettings["AWSBUCKET"];
return(client.ListVersions(new ListVersionsRequest().WithBucketName(BUCKET_NAME).WithKeyMarker(sObjectKey)));
}
public static void InvokeListBuckets()
{
NameValueCollection appConfig = ConfigurationManager.AppSettings;
IAmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(RegionEndpoint.USWest2);
try
{
ListBucketsResponse response = s3Client.ListBuckets();
int numBuckets = 0;
numBuckets = response.Buckets.Count;
Console.WriteLine("You have " + numBuckets + " Amazon S3 bucket(s).");
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode.Equals("InvalidAccessKeyId") ||
ex.ErrorCode.Equals("InvalidSecurity"))
{
Console.WriteLine("Please check the provided AWS access credentials.");
Console.WriteLine("If you haven't signed up for Amazon S3 yet, you can sign up at http://aws.amazon.com/s3.");
}
else
{
Console.WriteLine("Caught Exception: " + ex.Message);
Console.WriteLine("Response Status Code: " + ex.StatusCode);
Console.WriteLine("Error Code: " + ex.ErrorCode);
Console.WriteLine("Request ID: " + ex.RequestId);
}
}
Console.WriteLine();
}
public S3FileStorage()
{
var config = ConfigurationManager.AppSettings;
_bucketName = config["BucketName"];
_s3Client = AWSClientFactory.CreateAmazonS3Client(config["AWSAccessKey"], config["AWSSecretKey"], Amazon.RegionEndpoint.EUCentral1);
}
public static string CreateFileShare(string fileName, string fileContent)
{
try
{
var s3Client = AWSClientFactory.CreateAmazonS3Client();
String S3_KEY = string.Format("{0}{1}_{2}.pdf", AMAZONPublicFolder, fileName, Guid.NewGuid().ToString());
var request = new PutObjectRequest()
{
BucketName = AMAZONBucket,
Key = S3_KEY,
ContentBody = fileContent
};
s3Client.PutObject(request);
string preSignedURL = s3Client.GetPreSignedURL(new GetPreSignedUrlRequest()
{
BucketName = AMAZONBucket,
Key = S3_KEY,
Expires = System.DateTime.Now.AddMinutes(60 * 72)
});
return(preSignedURL);
}
catch (Amazon.S3.AmazonS3Exception ex)
{
throw;
}
catch (Exception e)
{
throw;
}
}
public static void Main(string[] args)
{
try
{
_s3Client = AWSClientFactory.CreateAmazonS3Client();
string path = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), BACKUP_JOB_FILE);
XmlSerializer serializer = new XmlSerializer(typeof(List <BackupJob>));
List <BackupJob> jobs = (List <BackupJob>)serializer.Deserialize(new FileStream(path, FileMode.Open));
foreach (var job in jobs.Where(p => p.Active))
{
ExecuteBackupJob(job);
_logger.Info("finished uploading: " + job.LocalRoot);
}
serializer.Serialize(new FileStream(path, FileMode.Create), jobs);
}
catch (Exception ex)
{
_logger.Error(ex.Message + " " + ex.StackTrace);
}
}
public static string PreSignedUrl(TemporaryAWSCredentials creds, string fileKey)
{
string url = "";
//var s3Client = new AmazonS3Client(new SessionAWSCredentials(creds.AccessKeyId, creds.SecretAccessKey, creds.Token)))
ResponseHeaderOverrides headerOverrides = new ResponseHeaderOverrides();
headerOverrides.ContentType = "application/pdf";
int secs = 0;
do
{
using (var s3Client = AWSClientFactory.CreateAmazonS3Client(GetAccesskey(), GetSecretkey()))
{
GetPreSignedUrlRequest request = new GetPreSignedUrlRequest()
.WithBucketName(GetBucketname())
.WithKey(fileKey.TrimStart('/'))
.WithProtocol(Protocol.HTTP)
.WithVerb(HttpVerb.GET)
//.WithResponseHeaderOverrides(headerOverrides)
.WithExpires(DateTime.Now.AddMinutes(120).AddSeconds(secs));
url = s3Client.GetPreSignedURL(request);
secs++;
}
} while ((url.Contains("%2B") || url.Contains("%2b") || url.Contains("+")) && secs < 30); // try again until a signature with no + sign is generated.
return(url);
}
/// <summary>
/// Copy the passed file to the S3 bucket
/// </summary>
/// <param name="Fqpn">Fully-qualified pathname of the local file. </param>
public static void CopyToS3(string Fqpn)
{
Globals.Log.InformationMessage("Begin copy file to URL: S3://{0}/{1}", AppSettingsImpl.Bucket.Value, MakeKeyName(Fqpn));
string accessKey = AppSettingsImpl.AccessKey.Value;
string secretKey = AppSettingsImpl.SecretKey.Value;
if (AppSettingsImpl.Encrypted.Value)
{
accessKey = Crypto.Unprotect(accessKey);
secretKey = Crypto.Unprotect(secretKey);
}
IAmazonS3 client = AWSClientFactory.CreateAmazonS3Client(accessKey, secretKey, RegionEndpoint.USEast1);
TransferUtility utility = new TransferUtility(client);
TransferUtilityUploadRequest request = new TransferUtilityUploadRequest()
{
BucketName = AppSettingsImpl.Bucket.Value,
Key = MakeKeyName(Fqpn),
FilePath = Fqpn
};
try
{
utility.Upload(request);
}
catch (Exception e)
{
throw new DownLoadException(string.Format("Attempt to upload file to S3 failed with exception: {0}", e.Message));
}
Globals.Log.InformationMessage("Copied file to S3");
}
/// <summary>
/// A Wrapper for the AWS.net SDK
/// </summary>
public S3Storage()
{
var accessKeyId = _dbContext.Key.SingleOrDefault(k => k.Name == "AccessKeyId").Data;
var secretAccessKey = _dbContext.Key.SingleOrDefault(k => k.Name == "SecretAccessKey").Data;
_client = AWSClientFactory.CreateAmazonS3Client(accessKeyId, secretAccessKey, RegionEndpoint.USEast1);
}
private IAmazonS3 GetClient()
{
var cfg = new AmazonS3Config {
UseHttp = true, MaxErrorRetry = 3, RegionEndpoint = RegionEndpoint.GetBySystemName(_region)
};
return(AWSClientFactory.CreateAmazonS3Client(_accessKeyId, _secretAccessKeyId, cfg));
}
public AwsHelper(string awsAccessKey, string awsSecretKey)
{
_awsAccessKey = awsAccessKey;
_awsSecretKey = awsSecretKey;
_s3Client = new Lazy <AmazonS3>(() => AWSClientFactory.CreateAmazonS3Client(_awsAccessKey, _awsSecretKey), true);
_cloudFront = new Lazy <AmazonCloudFront>(() => AWSClientFactory.CreateAmazonCloudFrontClient(_awsAccessKey, _awsSecretKey), true);
}
private AmazonS3 GetClient()
{
var cfg = new AmazonS3Config {
CommunicationProtocol = Protocol.HTTP, MaxErrorRetry = 3
};
return(AWSClientFactory.CreateAmazonS3Client(_accessKeyId, _secretAccessKeyId, cfg));
}
protected Amazon.S3.AmazonS3 GetS3Client(string accessKeyID, string secretAccessKey)
{
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(
accessKeyID,
secretAccessKey
);
return(s3Client);
}
public AwsS3Manager(string regionEndpoint, string accessKey, string secretAccessKey, string bucketName)
{
_bucketName = bucketName;
_accessKey = accessKey;
_secretAccessKey = secretAccessKey;
_uploadLinkLifeTimeSec = AwsS3Settings.Instance.LinksLifeTime.UploadLinkLifeTimeSec;
_downloadLinkLifeTimeSec = AwsS3Settings.Instance.LinksLifeTime.DownloadLinkLifeTimeSec;
_client = AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey, _regionEndpoints[regionEndpoint]);
}
public S3(String accesskey, String secretAccessKey, string bucket, string syncDir)
{
var config = new AmazonS3Config();
config.RegionEndpoint = RegionEndpoint.APNortheast1;
_bucket = bucket;
_s3Client = AWSClientFactory.CreateAmazonS3Client(accesskey, secretAccessKey, config);
_syncDir = syncDir;
}
public AWSStorageClient(string specificFolder)
{
_storageClient = AWSClientFactory.CreateAmazonS3Client();
S3DirectoryInfo rootDirectory = new S3DirectoryInfo(_storageClient, BucketName);
rootDirectory.Create();
_subDirectory = rootDirectory.CreateSubdirectory(specificFolder);
}
public S3(string accessKey, string secretKey, String serviceUrl)
{
var config = new AmazonS3Config()
{
ServiceURL = serviceUrl
};
_client = AWSClientFactory.CreateAmazonS3Client(accessKey, secretKey, config) as AmazonS3Client;
}
public override bool Execute()
{
RequireSecretKey();
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(AccessKey, SecretKey);
string bucketName = Bucket.ItemSpec;
Log.LogMessage(MessageImportance.High, "Connecting to \"{0}\"", bucketName);
WarnIfUneven(Tuple.Create("Puts", Puts), Tuple.Create("Keys", Keys));
foreach (var tuple in Zip(Puts, Keys))
{
ITaskItem put = tuple.Item1;
ITaskItem keyItem = tuple.Item2;
string key = keyItem.ItemSpec.Replace('\\', '/').TrimStart('/');
if (!put.Exists())
{
Log.LogMessage(MessageImportance.Normal, "Skipping {0} because it does not exist", key);
continue;
}
if (_Cancel)
{
return(false);
}
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
FilePath = put.ItemSpec,
Key = key,
Timeout = -1,
ReadWriteTimeout = 300000 // 5 minutes in milliseconds
};
S3CannedACL cannedACL;
if (Enum.TryParse <S3CannedACL>(put.GetMetadata("CannedACL") ?? "", out cannedACL))
{
Log.LogMessage(MessageImportance.Normal, "Applying CannedACL: {0}", cannedACL);
putObjectRequest.CannedACL = cannedACL;
}
string contentType = put.GetMetadata("ContentType");
if (!string.IsNullOrWhiteSpace(contentType))
{
Log.LogMessage(MessageImportance.Normal, "Applying ContentType: {0}", contentType);
putObjectRequest.ContentType = contentType;
}
Log.LogMessage(MessageImportance.High, "Putting \"{0}\"", key);
using (var upload = s3Client.PutObject(putObjectRequest))
{
}
}
return(true);
}
/// <summary>
/// Initializes a new instance of the <see cref="AmazonS3VirtualFile"/> class.
/// </summary>
/// <param name="provider">The provider.</param>
/// <param name="virtualPath">The virtual path.</param>
public AmazonS3VirtualFile(AmazonS3VirtualPathProvider provider, string virtualPath) : base(virtualPath)
{
_provider = provider;
_virtualPath = virtualPath;
this._client = AWSClientFactory.CreateAmazonS3Client(new AmazonS3Config
{
ServiceURL = "s3.amazonaws.com",
CommunicationProtocol = Protocol.HTTP
});
}
public AWSStorageClientClient(string directory)
{
_storageClient = AWSClientFactory.CreateAmazonS3Client();
S3DirectoryInfo rootDirectory = new S3DirectoryInfo(_storageClient, BucketName);
rootDirectory.Create();
_mainDirectory = rootDirectory.CreateSubdirectory(directory);
_outputDirectory = rootDirectory.CreateSubdirectory("AnalysisOutput");
}