private byte[] Decompress(byte[] input)
{
byte[] retVal = null;
switch (this.chunkCompressionType)
{
case CompressionType.None:
retVal = input;
break;
case CompressionType.GZip:
retVal = AzureHelper.GZipDecompress(input);
break;
case CompressionType.BZip2:
retVal = AzureHelper.BZip2Decompress(input);
break;
default:
throw new NotImplementedException("compression type " + this.chunkCompressionType + " not implemented");
}
return(retVal);
}
public bool Sync()
{
bool retVal = true;
try
{
if (syncDirection == SynchronizeDirection.Upload)
{
if (logger != null)
{
logger.Log("Start Synchronizer Enlisting Files");
}
List <string> fileList = AzureHelper.ListFiles(FqDirName, this.indexFileName, this.dataFileName); //what if this returns empty list
List <string> fileListToUpload = new List <string>(fileList);
if (logger != null)
{
logger.Log("End Synchronizer Enlisting Files");
}
// todo filter list, to exclude archival stream files.
IWorkItemResult[] workItemResults = new IWorkItemResult[this.ThreadPoolSize];
int i = 0;
foreach (string file in fileList)
{
if (Path.GetFileName(file).Equals(this.indexFileName))
{
if (logger != null)
{
logger.Log("Start Synchronizer Upload FileSimple");
}
retVal = (azureHelper.UploadFileToBlockBlob(indexFileName, file)) && retVal;
if (logger != null)
{
logger.Log("End Synchronizer Upload FileSimple");
}
fileListToUpload.Remove(file);
}
else if (Path.GetFileName(file).Equals(this.dataFileName))
{
if (logger != null)
{
logger.Log("Start Synchronizer Upload FileAsChunks");
}
chunkListHash = azureHelper.UploadFileAsChunks(file);
if (logger != null)
{
logger.Log("End Synchronizer Upload FileAsChunks");
}
retVal = ((chunkListHash == null) ? false : true) && retVal;
fileListToUpload.Remove(file);
}
if (SyncFactory.FilesExcludedFromSync.Contains(Path.GetFileName(file)) || SyncFactory.FilesExcludedFromSync.Contains(Path.GetExtension(file)))
{
// AzureHelper.structuredLog("I", "Ignoring sync for file {0}" , file);
// do nothing. just ignore the file
fileListToUpload.Remove(file);
}
} //we've taken care of the index, data, and files to ignore. whatever is left are datablock files for file data streams (?)
if (logger != null)
{
logger.Log("Start Synchronizer Upload FDSFiles");
}
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = this.ThreadPoolSize;
foreach (string file in fileListToUpload)
{
workItemResults[i++] = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadFileToBlockBlob_worker), file);
}
threadPool.Start();
threadPool.WaitForIdle();
for (int j = 0; j < i; j++)
{
retVal = retVal && ((bool)workItemResults[j].Result);
}
threadPool.Shutdown();
if (logger != null)
{
logger.Log("End Synchronizer Upload FDSFiles");
}
}
if (syncDirection == SynchronizeDirection.Download)
{
this.GetDataFileMetadata(); // get chunkMD and populate chunkList Hash
retVal = azureHelper.DownloadBlockBlobToFile(indexFileName, FqDirName + "/" + indexFileName); // download index
}
}
catch (Exception e)
{
if (logger != null)
{
logger.Log("Exception in Sync(): " + e.GetType() + ", " + e.Message);
}
retVal = false;
}
return(retVal);
}
public bool Sync()
{
if (syncDirection == SynchronizeDirection.Upload)
{
if (logger != null)
{
logger.Log("Start Synchronizer Enlisting Files");
}
List <string> fileList = AzureHelper.ListFiles(localSource, this.indexFileName, this.dataFileName);
List <string> fileListToUpload = new List <string>(fileList);
if (logger != null)
{
logger.Log("End Synchronizer Enlisting Files");
}
// todo filter list, to exclude archival stream files.
bool retVal = true;
foreach (string file in fileList)
{
if (Path.GetFileName(file).Equals(this.indexFileName))
{
if (logger != null)
{
logger.Log("Start Synchronizer Upload FileSimple");
}
retVal = amazonS3Helper.UploadFileToS3Object(indexFileName, file);
if (logger != null)
{
logger.Log("End Synchronizer Upload FileSimple");
}
fileListToUpload.Remove(file);
}
else if (Path.GetFileName(file).Equals(this.dataFileName))
{
if (logger != null)
{
logger.Log("Start Synchronizer Upload FileAsChunks");
}
chunkListHash = amazonS3Helper.UploadFileAsChunks(file);
if (logger != null)
{
logger.Log("End Synchronizer Upload FileAsChunks");
}
retVal = (chunkListHash == null) ? false : true;
fileListToUpload.Remove(file);
}
if (SyncFactory.FilesExcludedFromSync.Contains(Path.GetFileName(file)) || SyncFactory.FilesExcludedFromSync.Contains(Path.GetExtension(file)))
{
// AzureHelper.structuredLog("I", "Ignoring sync for file {0}" , file);
// do nothing. just ignore the file
fileListToUpload.Remove(file);
}
} //we've taken care of the index, data, and files to ignore. whatever is left are datablock files for file data streams (?)
if (logger != null)
{
logger.Log("Start Synchronizer Upload FDSFiles");
}
SmartThreadPool threadPool = new SmartThreadPool();
threadPool.MaxThreads = this.MaxConcurrentFileSyncThreads;
foreach (string file in fileListToUpload)
{
IWorkItemResult wir1 = threadPool.QueueWorkItem(new WorkItemCallback(this.UploadFileToBlockBlob_worker), file);
}
threadPool.Start();
threadPool.WaitForIdle();
threadPool.Shutdown();
if (logger != null)
{
logger.Log("End Synchronizer Upload FDSFiles");
}
}
if (syncDirection == SynchronizeDirection.Download)
{
this.GetDataFileMetadata(); // get chunkMD and populate chunkList Hash
return(amazonS3Helper.DownloadS3ObjectToFile(indexFileName, localSource + "/" + indexFileName)); // download index
}
return(true);//TODO fix this to return correct value for upload cases
}