public Tuple <List <ChunkInfo>, byte[]> GetBlobMetadata(string blobName)
{
CloudBlockBlob chunkMDblockBlob = GetBlockBlobReference(ChunkMetadataBlobPrefix + blobName);
bool blobExists = BlockBlobExists(chunkMDblockBlob);
List <ChunkInfo> retVal;
byte[] hash;
if (blobExists)
{
string chunkMD_JSON = chunkMDblockBlob.DownloadText();
FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMD_JSON);
SHA1 sha1 = new SHA1CryptoServiceProvider();
hash = sha1.ComputeHash(Encoding.ASCII.GetBytes(chunkMD_JSON));
retVal = fileMD.ChunkList;
}
else
{
retVal = null;
hash = null;
if (!blobName.Equals("stream_md.dat"))
{
Console.WriteLine("Now: blob exists false for " + chunkMDblockBlob);
}
}
return(new Tuple <List <ChunkInfo>, byte[]>(retVal, hash));
}
public System.Tuple <List <ChunkInfo>, byte[]> GetObjectMetadata(string s3objectName)
{
List <ChunkInfo> retVal = null;
byte[] hash = null;
string metadataObjectName = ChunkMetadataObjectPrefix + s3objectName;
bool s3ObjectExists = S3ObjectExists(metadataObjectName);
if (s3ObjectExists)
{
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(metadataObjectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
StreamReader reader = new StreamReader(response.ResponseStream);
string chunkMD_JSON = reader.ReadToEnd();
FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMD_JSON);
SHA1 sha1 = new SHA1CryptoServiceProvider();
hash = sha1.ComputeHash(Encoding.ASCII.GetBytes(chunkMD_JSON));
retVal = fileMD.ChunkList;
}
return(new System.Tuple <List <ChunkInfo>, byte[]>(retVal, hash));
}
public byte[] UploadFileAsChunks(string filePath)
{
string blobName;
List <ChunkInfo> chunkList_cloud = new List <ChunkInfo>();; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
List <ChunkInfo> chunkList_local; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
try
{
// First check if chunkMD exists on the server or not
if (logger != null)
{
logger.Log("Start Synchronizer Check Blob Exists");
}
blobName = Path.GetFileName(filePath);
CloudBlockBlob chunkMDblockBlob = GetBlockBlobReference(ChunkMetadataBlobPrefix + blobName);
bool blobExists = BlockBlobExists(chunkMDblockBlob);
if (logger != null)
{
logger.Log("End Synchronizer Check Blob Exists");
}
if (blobExists)
{
if (logger != null)
{
logger.Log("Start Synchronizer Fill Remote ChunkList");
}
// Fill chunkList
FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMDblockBlob.DownloadText());
StaticChunkSize = fileMD.StaticChunkSize;
// Get chunklist at cloud in memory locally
chunkList_cloud = fileMD.ChunkList;
if (logger != null)
{
logger.Log("End Synchronizer Fill Remote ChunkList");
}
chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType);
chunkEncryptionType = SyncFactory.GetEncryptionType(fileMD.encryptionType);
}
if (logger != null)
{
logger.Log("Start Synchronizer Fill Local ChunkList");
}
StaticChunk staticChunker = new StaticChunk(StaticChunkSize);
// Store local chunkList in memory
//long start = DateTime.Now.Ticks;
chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface
//long end = DateTime.Now.Ticks;
// Console.WriteLine("time taken : " + (double)((double)(end - start) / (double)10000000));
if (logger != null)
{
logger.Log("End Synchronizer Fill Local ChunkList");
}
// structuredLog("I", "Number of chunks locally: " + chunkList_local.Count);
// Figure out the changes
if (logger != null)
{
logger.Log("Start Synchronizer ChunkList Compare");
}
List <ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud);
// structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count);
// structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count);
if (logger != null)
{
logger.Log("End Synchronizer ChunkList Compare");
}
if (logger != null)
{
logger.Log("Start Synchronizer Upload Multiple Chunks");
}
UploadChunkList(ref chunkList_toUpload, filePath, blobName);
if (logger != null)
{
logger.Log("End Synchronizer Upload Multiple Chunks");
}
// structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count);
if (logger != null)
{
logger.Log("Start Synchronizer Commit BlockList");
}
// Commit the ordered blocklist
if (chunkList_toUpload.Count > 0)// write new metadata, and putblocklist() if we uploaded some chunks
{
CloudBlockBlob blockBlob = GetBlockBlobReference(blobName);
List <int> blockIDCommitList = GetBlockIDList(ref chunkList_local, ref chunkList_cloud, ref chunkList_toUpload);
long startt = DateTime.Now.Ticks;
blockBlob.PutBlockList(EncodeBlockList(blockIDCommitList), GetBlobRequestOptions());
long endt = DateTime.Now.Ticks;
if (logger != null)
{
logger.Log("PUTBLOCK LIST : " + (double)((double)(endt - startt) / (double)10000000));
}
}
if (logger != null)
{
logger.Log("End Synchronizer Commit BlockList");
}
// Upload the local chunklist to the cloud
if (logger != null)
{
logger.Log("Start Synchronizer ChunkList Upload");
}
string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter());
if (chunkList_toUpload.Count > 0 || chunkList_local.Count == 0) //upload new chunk list only if we uploaded some new chunks, or if this is a new stream, with no data in stream.dat
{
chunkMDblockBlob.UploadText(json);
}
SHA1 sha1 = new SHA1CryptoServiceProvider();
byte[] ret = sha1.ComputeHash(Encoding.ASCII.GetBytes(json));
if (logger != null)
{
logger.Log("End Synchronizer ChunkList Upload");
}
return(ret);
}
catch (Exception e)
{
structuredLog("E", " . UploadFileAsChunks: " + e);
return(null);
}
}
public byte[] UploadFileAsChunks(string filePath)
{
string s3objectName;
List <ChunkInfo> chunkList_cloud = new List <ChunkInfo>();; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
List <ChunkInfo> chunkList_local; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
try
{
if (logger != null)
{
logger.Log("Start Synchronizer Check Blob Exists");
}
s3objectName = Path.GetFileName(filePath);
bool s3ObjectExists = S3ObjectExists(ChunkMetadataObjectPrefix + s3objectName);
if (logger != null)
{
logger.Log("End Synchronizer Check Blob Exists");
}
if (s3ObjectExists)
{
if (logger != null)
{
logger.Log("Start Synchronizer Fill Remote ChunkList");
}
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(bucketName);
request.WithKey(ChunkMetadataObjectPrefix + s3objectName);
GetObjectResponse response = amazonS3Client.GetObject(request);
StreamReader reader = new StreamReader(response.ResponseStream);
string chunkMD_JSON = reader.ReadToEnd();
FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMD_JSON);
StaticChunkSize = fileMD.StaticChunkSize;
chunkList_cloud = fileMD.ChunkList;
if (logger != null)
{
logger.Log("End Synchronizer Fill Remote ChunkList");
}
chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType);
chunkEncryptionType = SyncFactory.GetEncryptionType(fileMD.encryptionType);
}
if (logger != null)
{
logger.Log("Start Synchronizer Fill Local ChunkList");
}
StaticChunk staticChunker = new StaticChunk(StaticChunkSize);
chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface
// structuredLog("I", "Number of chunks locally: " + chunkList_local.Count);
if (logger != null)
{
logger.Log("End Synchronizer Fill Local ChunkList");
}
if (logger != null)
{
logger.Log("Start Synchronizer ChunkList Compare");
}
List <ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud);
// structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count);
// structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count);
if (logger != null)
{
logger.Log("End Synchronizer ChunkList Compare");
}
if (logger != null)
{
logger.Log("Start Synchronizer Upload Multiple Chunks");
}
UploadChunkList(ref chunkList_toUpload, filePath, s3objectName);
if (logger != null)
{
logger.Log("End Synchronizer Upload Multiple Chunks");
}
// structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count);
if (logger != null)
{
logger.Log("Start Synchronizer ChunkList Upload");
}
string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter());
if (chunkList_toUpload.Count > 0) //upload new chunk list only if we uploaded some new chunks
{
UploadStringToS3Object(ChunkMetadataObjectPrefix + s3objectName, json);
}
SHA1 sha1 = new SHA1CryptoServiceProvider();
byte[] ret = sha1.ComputeHash(Encoding.ASCII.GetBytes(json));
if (logger != null)
{
logger.Log("End Synchronizer ChunkList Upload");
}
return(ret);
}
catch (Exception e)
{
structuredLog("E", " . UploadFileAsChunks: " + e);
return(null);
}
}