private object UploadChunk_Worker(object state)
{
Tuple <string, string, ChunkInfo> rx = (Tuple <string, string, ChunkInfo>)state;
string filePath = rx.Item1;
string blobName = rx.Item2;
ChunkInfo chunk = rx.Item3;
UploadChunk(filePath, blobName, ref chunk);
return(null);
}
private void UploadChunkList(ref List <ChunkInfo> chunkList_toUpload, string filePath, string blobName)
{
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = MaxConcurrentUploadThreads;
foreach (ChunkInfo chunk in chunkList_toUpload)
{
ChunkInfo chunkToUpload = chunk;
IWorkItemResult wir1 = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadChunk_Worker), new Tuple <string, string, ChunkInfo>(filePath, blobName, chunkToUpload));
//UploadChunk(filePath, blobName, ref chunkToUpload);
}
threadPool.Start();
threadPool.WaitForIdle();
threadPool.Shutdown();
}
private void UploadChunk(string filePath, string blobName, ref ChunkInfo chunk)
{
int tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
if (logger != null)
{
logger.Log("Start Synchronizer Read Chunk From File" + tid);
}
if (StaticChunkSize > MaxAzureBlockSize && StaticChunkSize % MaxAzureBlockSize != 0)
{
throw new NotImplementedException("chunk size (current: " + chunk.rsize + ") has to be a fixed multiple of maximum block size :" + MaxAzureBlockSize);
}
// structuredLog("I", "uploading chunk with index: " + chunk.chunkIndex);
byte[] chunkBuffer = new byte[chunk.rsize];
long start = DateTime.Now.Ticks;
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
fs.Seek(chunk.roffset, SeekOrigin.Begin);
BinaryReader br = new BinaryReader(fs);
br.Read(chunkBuffer, 0, chunkBuffer.Length);//read chunk into buffer
fs.Close();
}
long end = DateTime.Now.Ticks;
// Console.WriteLine("time taken : " + (double)((double)(end - start) / (double)10000000));
if (logger != null)
{
logger.Log("End Synchronizer Read Chunk From File" + tid);
}
// Here is the compression and encryption for the chunk
if (logger != null)
{
logger.Log("Start Synchronizer Compress Chunk");
}
byte[] compressedChunkBuffer = Compress(chunkBuffer);// for now
if (logger != null)
{
logger.Log("End Synchronizer Compress Chunk");
}
if (logger != null)
{
logger.Log("Start Synchronizer Encrypt Chunk");
}
byte[] encryptedCompressedChunkBuffer = Encrypt(compressedChunkBuffer);
if (logger != null)
{
logger.Log("End Synchronizer Encrypt Chunk");
}
if (logger != null)
{
logger.Log("Start Synchronizer Upload Chunk" + tid);
}
int blockID;
if (StaticChunkSize < MaxAzureBlockSize)
{
blockID = (int)chunk.chunkIndex;
}
else
{
blockID = (int)chunk.chunkIndex * (int)(StaticChunkSize / MaxAzureBlockSize);
}
int blockCount = 0;
CloudBlockBlob blockBlob = GetBlockBlobReference(blobName);
// Upload the chunk
while (blockCount * MaxAzureBlockSize < encryptedCompressedChunkBuffer.Length)
{
string blockIdBase64 = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(blockID.ToString(CultureInfo.InvariantCulture).PadLeft(16, '0')));
int blockSize = MaxAzureBlockSize <= encryptedCompressedChunkBuffer.Length - blockCount * MaxAzureBlockSize ? MaxAzureBlockSize : encryptedCompressedChunkBuffer.Length - blockCount * MaxAzureBlockSize;
long startt = DateTime.Now.Ticks;
using (MemoryStream ms = new MemoryStream(encryptedCompressedChunkBuffer, blockCount * MaxAzureBlockSize, blockSize))
{
blockBlob.PutBlock(blockIdBase64, ms, GetMD5FromStream(ms.ToArray()), GetBlobRequestOptions());
}
long endt = DateTime.Now.Ticks;
//Console.WriteLine("+" + (double)((double)(endt - startt) / (double)10000000));
chunk.AddToBlockList(blockID);
chunk.SetCSize(encryptedCompressedChunkBuffer.Length);
blockID++;
blockCount++;
}
if (logger != null)
{
logger.Log("End Synchronizer Upload Chunk" + tid);
}
}
private void UploadChunk(string filePath, string s3objectName, ref ChunkInfo chunk)
{
// structuredLog("I", "uploading chunk with index: " + chunk.chunkIndex);
// if (logger != null) logger.Log("Thread: " + System.Threading.Thread.CurrentThread.ManagedThreadId);
if (logger != null)
{
logger.Log("Start Synchronizer Read Chunk From File");
}
byte[] chunkBuffer = new byte[chunk.rsize];
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
fs.Seek(chunk.roffset, SeekOrigin.Begin);
BinaryReader br = new BinaryReader(fs);
br.Read(chunkBuffer, 0, chunkBuffer.Length);//read chunk into buffer
fs.Close();
}
if (logger != null)
{
logger.Log("End Synchronizer Read Chunk From File");
}
// Here is the compression and encryption for the chunk
if (logger != null)
{
logger.Log("Start Synchronizer Compress Chunk");
}
byte[] compressedChunkBuffer = Compress(chunkBuffer);// for now
if (logger != null)
{
logger.Log("End Synchronizer Compress Chunk");
}
if (logger != null)
{
logger.Log("Start Synchronizer Encrypt Chunk");
}
byte[] encryptedCompressedChunkBuffer = Encrypt(compressedChunkBuffer);
if (logger != null)
{
logger.Log("End Synchronizer Encrypt Chunk");
}
if (logger != null)
{
logger.Log("Start Synchronizer Upload Chunk");
}
string chunkObjectName = ChunkObjectNamePrefix + s3objectName + "-" + chunk.chunkIndex;
if (UploadByteArrayToS3Object(chunkObjectName, encryptedCompressedChunkBuffer))
{
chunk.SetBlobName(chunkObjectName);
chunk.SetCSize(encryptedCompressedChunkBuffer.Length);
}
else
{
throw new Exception("Chunk upload for given chunk has failed. FileName: " + s3objectName + " . Chunk:" + chunk.ToString());
}
if (logger != null)
{
logger.Log("End Synchronizer Upload Chunk");
}
}
