public RackspaceCloudFilesSynchronizer(RemoteInfo ri, string container, SynchronizeDirection syncDirection)
{
this.disposed = false;
this.username = ri.accountName;
this.apiKey = ri.accountKey;
this.syncDirection = syncDirection;
this.container = container;
try
{
var cloudIdentity = new CloudIdentity()
{
APIKey = this.apiKey, Username = this.username
};
var cloudFilesProvider = new CloudFilesProvider(cloudIdentity);
ObjectStore createContainerResponse = cloudFilesProvider.CreateContainer(container);// assume default region for now
if (!createContainerResponse.Equals(ObjectStore.ContainerCreated) && !createContainerResponse.Equals(ObjectStore.ContainerExists))
{
Console.WriteLine("Container creation failed! Response: " + createContainerResponse.ToString());
}
}
catch (Exception e)
{
Console.WriteLine("Exception in creating container: " + e);
}
}
/// <summary>
/// We-use the remote info as: accountName = awsAccessKeyId and accountKey = awsSecretAccessKey
/// </summary>
public AmazonS3Synchronizer(RemoteInfo remoteInfo, string bucket, SynchronizeDirection syncDirection, CompressionType compressionType, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize = 1 )
{
this.logger = log;
disposed = false;
this.syncDirection = syncDirection;
bucketName = bucket.ToString().Replace(' ', '-').ToLower(); ;// amazon S3 does not like spaces in bucket names
amazonS3Helper = new AmazonS3Helper(remoteInfo, bucket, compressionType, encryptionType, encryptionKey, initializationVector, logger, ChunkSize, ThreadPoolSize);
this.MaxConcurrentFileSyncThreads = ThreadPoolSize;
}
/// <summary>
/// We-use the remote info as: accountName = awsAccessKeyId and accountKey = awsSecretAccessKey
/// </summary>
public AmazonS3Synchronizer(RemoteInfo remoteInfo, string bucket, SynchronizeDirection syncDirection, CompressionType compressionType, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize = 1)
{
this.logger = log;
disposed = false;
this.syncDirection = syncDirection;
bucketName = bucket.ToString().Replace(' ', '-').ToLower();;// amazon S3 does not like spaces in bucket names
amazonS3Helper = new AmazonS3Helper(remoteInfo, bucket, compressionType, encryptionType, encryptionKey, initializationVector, logger, ChunkSize, ThreadPoolSize);
this.MaxConcurrentFileSyncThreads = ThreadPoolSize;
}
//protected SyncOrchestrator orchestrator;
public AzureChunkSynchronizer(RemoteInfo ri, string container, SynchronizeDirection syncDirection, CompressionType compressionType, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize = 1)
{
logger = log;
disposed = false;
// Setup Store and Provider
//
this.accountName = ri.accountName;
this.accountKey = ri.accountKey;
this.container = container;
this.syncDirection = syncDirection;
this.azureHelper = new AzureHelper(this.accountName, this.accountKey, this.container, compressionType, encryptionType, encryptionKey, initializationVector, log, ChunkSize, ThreadPoolSize);
this.chunkListHash = null;
this.ThreadPoolSize = ThreadPoolSize;
}
//protected SyncOrchestrator orchestrator;
public AzureChunkSynchronizer(RemoteInfo ri, string container, SynchronizeDirection syncDirection, CompressionType compressionType, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize=1)
{
logger = log;
disposed = false;
// Setup Store and Provider
//
this.accountName = ri.accountName;
this.accountKey = ri.accountKey;
this.container = container;
this.syncDirection = syncDirection;
this.azureHelper = new AzureHelper(this.accountName, this.accountKey, this.container, compressionType, encryptionType, encryptionKey, initializationVector, log, ChunkSize, ThreadPoolSize);
this.chunkListHash = null;
this.ThreadPoolSize = ThreadPoolSize;
}
public AzureHelper(string accountName, string accountKey, string containerName, CompressionType chunkCompression, EncryptionType chunkEncryption, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize)
{
this.logger = log;
//this.azureBlobType = AzureBlobType.BlockBlob;
//this.mapping = Mapping.FileToBlob;
this.remoteInfo = new RemoteInfo(accountName, accountKey);
this.containerName = containerName;
this.chunkCompressionType = chunkCompression;
this.chunkEncryptionType = chunkEncryption;
this.encryptionKey = encryptionKey;
this.InitializationVector = initializationVector;
this.StaticChunkSize = ChunkSize;
this.MaxConcurrentUploadThreads = ThreadPoolSize;
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
this.remoteInfo = null;
this.bucketName = null;
this.encryptionKey = null;
this.InitializationVector = null;
amazonS3Client = null;
disposed = true;
}
}
}
public AmazonS3Helper(RemoteInfo remoteInfo, string bucketName, CompressionType chunkCompression, EncryptionType chunkEncryption , byte[] encryptionKey , byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize)
{
this.logger = log;
this.remoteInfo = remoteInfo;
this.bucketName = bucketName;
this.chunkCompressionType = chunkCompression;
this.chunkEncryptionType = chunkEncryption;
this.encryptionKey = encryptionKey;
this.InitializationVector = initializationVector;
amazonS3Client = new AmazonS3Client(remoteInfo.accountName, remoteInfo.accountKey);
this.StaticChunkSize = ChunkSize;
this.MaxConcurrentUploadThreads = ThreadPoolSize;
this.disposed = false;
CreateBucket(this.bucketName);
}
public AmazonS3Helper(RemoteInfo remoteInfo, string bucketName, CompressionType chunkCompression, EncryptionType chunkEncryption, byte[] encryptionKey, byte[] initializationVector, Logger log, int ChunkSize, int ThreadPoolSize)
{
this.logger = log;
this.remoteInfo = remoteInfo;
this.bucketName = bucketName;
this.chunkCompressionType = chunkCompression;
this.chunkEncryptionType = chunkEncryption;
this.encryptionKey = encryptionKey;
this.InitializationVector = initializationVector;
amazonS3Client = new AmazonS3Client(remoteInfo.accountName, remoteInfo.accountKey);
this.StaticChunkSize = ChunkSize;
this.MaxConcurrentUploadThreads = ThreadPoolSize;
this.disposed = false;
CreateBucket(this.bucketName);
}
public void AzureChunkWriteRead()
{
string accountName = "testdrive";
string accountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q==";
RemoteInfo ri = new RemoteInfo(accountName, accountKey);
AzureHelper helper = new AzureHelper(accountName, accountKey, "testuploadchunksfile", CompressionType.None, EncryptionType.None, null, null, new Logger(), 4*1024*1024 , 1);
helper.UploadFileAsChunks("D:\\testfiles\\testhuge.txt");
int OFFSET_TO_READ = 4492321;
int BYTES_TO_READ = 11000;
List<ChunkInfo> metadata = helper.GetBlobMetadata("testhuge.txt").Item1;
Dictionary<int, long> chunkindexandoffsets = helper.GetChunkIndexAndOffsetInChunk(metadata, OFFSET_TO_READ, BYTES_TO_READ);
byte[] temp = null;
foreach(int chunkIndex in chunkindexandoffsets.Keys)
{
if(temp!=null)
temp = temp.Concat(helper.DownloadChunk("testhuge.txt", metadata,chunkIndex)).ToArray();
else
temp = helper.DownloadChunk("testhuge.txt", metadata, chunkIndex);
}
byte[] test = temp.Skip((int)chunkindexandoffsets.ElementAt(0).Value).Take(BYTES_TO_READ).ToArray();
byte[] truth = new byte[BYTES_TO_READ];
using (BinaryReader reader = new BinaryReader(new FileStream("D:\\testfiles\\testhuge.txt", FileMode.Open)))
{
reader.BaseStream.Seek(OFFSET_TO_READ, SeekOrigin.Begin);
reader.Read(truth, 0, BYTES_TO_READ);
}
bool arraysAreEqual = Enumerable.SequenceEqual(test, truth);
Console.WriteLine(arraysAreEqual);
if (!arraysAreEqual)
throw new Exception("local and downloaded bits dont match");
}
public void AmazonS3ChunkWriteRead()
{
string accountName = "";
string accountKey = "";
RemoteInfo ri = new RemoteInfo(accountName, accountKey);
AmazonS3Helper helper = new AmazonS3Helper(new RemoteInfo(accountName, accountKey), "testupl0", CompressionType.GZip, EncryptionType.None, null, null, new Logger(), 4*1024*1024, 1);
helper.UploadFileAsChunks("D:\\testfiles\\test.txt");
int OFFSET_TO_READ = 4492321;
int BYTES_TO_READ = 11000;
List<ChunkInfo> metadata = helper.GetObjectMetadata("test.txt").Item1;
Dictionary<int, long> chunkindexandoffsets = helper.GetChunkIndexAndOffsetInChunk(metadata, OFFSET_TO_READ, BYTES_TO_READ);
byte[] temp = null;
foreach (int chunkIndex in chunkindexandoffsets.Keys)
{
if (temp != null)
temp = temp.Concat(helper.DownloadChunk("test.txt", chunkIndex)).ToArray();
else
temp = helper.DownloadChunk("test.txt", chunkIndex);
}
byte[] test = temp.Skip((int)chunkindexandoffsets.ElementAt(0).Value).Take(BYTES_TO_READ).ToArray();
byte[] truth = new byte[BYTES_TO_READ];
using (BinaryReader reader = new BinaryReader(new FileStream("D:\\testfiles\\test.txt", FileMode.Open)))
{
reader.BaseStream.Seek(OFFSET_TO_READ, SeekOrigin.Begin);
reader.Read(truth, 0, BYTES_TO_READ);
}
bool arraysAreEqual = Enumerable.SequenceEqual(test, truth);
Console.WriteLine(arraysAreEqual);
if (!arraysAreEqual)
throw new Exception("local and downloaded bits dont match");
}
public RackspaceCloudFilesSynchronizer(RemoteInfo ri, string container, SynchronizeDirection syncDirection)
{
this.disposed = false;
this.username = ri.accountName;
this.apiKey = ri.accountKey;
this.syncDirection = syncDirection;
this.container = container;
try
{
var cloudIdentity = new CloudIdentity() { APIKey = this.apiKey, Username = this.username };
var cloudFilesProvider = new CloudFilesProvider(cloudIdentity);
ObjectStore createContainerResponse = cloudFilesProvider.CreateContainer(container);// assume default region for now
if (!createContainerResponse.Equals(ObjectStore.ContainerCreated) && !createContainerResponse.Equals(ObjectStore.ContainerExists))
Console.WriteLine("Container creation failed! Response: " + createContainerResponse.ToString());
}
catch (Exception e)
{
Console.WriteLine("Exception in creating container: " + e);
}
}
public void AzureChunkWriteRead()
{
string accountName = "msrlot";
string accountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q==";
RemoteInfo ri = new RemoteInfo(accountName, accountKey);
AzureHelper helper = new AzureHelper(accountName, accountKey, "testuploadchunksfile", CompressionType.None, EncryptionType.None, null, null, new Logger(), 4*1024*1024 , 1);
helper.UploadFileAsChunks("D:\\testfiles\\testhuge.txt");
int OFFSET_TO_READ = 4492321;
int BYTES_TO_READ = 11000;
List<ChunkInfo> metadata = helper.GetBlobMetadata("testhuge.txt").Item1;
Dictionary<int, long> chunkindexandoffsets = helper.GetChunkIndexAndOffsetInChunk(metadata, OFFSET_TO_READ, BYTES_TO_READ);
byte[] temp = null;
foreach(int chunkIndex in chunkindexandoffsets.Keys)
{
if(temp!=null)
temp = temp.Concat(helper.DownloadChunk("testhuge.txt", metadata,chunkIndex)).ToArray();
else
temp = helper.DownloadChunk("testhuge.txt", metadata, chunkIndex);
}
byte[] test = temp.Skip((int)chunkindexandoffsets.ElementAt(0).Value).Take(BYTES_TO_READ).ToArray();
byte[] truth = new byte[BYTES_TO_READ];
using (BinaryReader reader = new BinaryReader(new FileStream("D:\\testfiles\\testhuge.txt", FileMode.Open)))
{
reader.BaseStream.Seek(OFFSET_TO_READ, SeekOrigin.Begin);
reader.Read(truth, 0, BYTES_TO_READ);
}
bool arraysAreEqual = Enumerable.SequenceEqual(test, truth);
Console.WriteLine(arraysAreEqual);
if (!arraysAreEqual)
throw new Exception("local and downloaded bits dont match");
}
public AzureSynchronizer(RemoteInfo ri, string container, SynchronizeDirection syncDirection)
{
disposed = false;
string _containerName = container;
//
// Setup Store and Provider
//
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(ri.accountName, ri.accountKey), true);
AzureBlobStore blobStore = new AzureBlobStore(_containerName, storageAccount);
Console.WriteLine("Successfully created/attached to container {0}.", _containerName);
AzureBlobSyncProvider azureProvider = new AzureBlobSyncProvider(_containerName, blobStore);
azureProvider.ApplyingChange += new EventHandler<ApplyingBlobEventArgs>(UploadingFile);
orchestrator = new SyncOrchestrator();
orchestrator.RemoteProvider = azureProvider;
if (syncDirection == SynchronizeDirection.Upload)
orchestrator.Direction = SyncDirectionOrder.Upload;
else if (syncDirection == SynchronizeDirection.Download)
orchestrator.Direction = SyncDirectionOrder.Download;
}
private ISync CreateAzureSynchronizer(RemoteInfo ri, string container, Logger log, SynchronizeDirection syncDirection, CompressionType compressionType,int ChunkSizeForUpload, int ThreadPoolSize, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector)
{
return new AzureChunkSynchronizer(ri, container, syncDirection, compressionType, encryptionType, encryptionKey, initializationVector, log, ChunkSizeForUpload, ThreadPoolSize);
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
this.remoteInfo = null;
this.bucketName = null;
this.encryptionKey = null;
this.InitializationVector = null;
amazonS3Client = null;
disposed = true;
}
}
}
private ISync CreateAmazonS3Synchronizer(RemoteInfo ri, string container, Logger log, SynchronizeDirection syncDirection, CompressionType compressionType, int ChunkSizeForUpload, int ThreadPoolSize, EncryptionType encryptionType, byte[] encryptionKey, byte[] initializationVector)
{
return(new AmazonS3Synchronizer(ri, container, syncDirection, compressionType, encryptionType, encryptionKey, initializationVector, log, ChunkSizeForUpload, ThreadPoolSize));
}
