private void MoveWorkToNextQueue()
{
opts.WorkerId++;
if (opts.WorkerId >= opts.WorkerCount)
{
opts.WorkerId = 0;
}
int batchIndex = GetBatchStartingIndex(topLevelFoldersCount, opts);
Log.Always("BATCH INDEX " + batchIndex);
Log.Always(FixedStrings.ProcessingQueue + batchIndex);
if (topLevelFoldersCount > opts.WorkerCount)
{
// We have more folders than workers, we assign queues based on ThreadId
folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + opts.WorkerId);
fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + opts.WorkerId);
}
else
{
// We have more workers than folders, we assign queues based on zero based folder index
folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + batchIndex);
fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + batchIndex);
}
}
/// <summary>
/// Starts the processing of work given the path provided to the tool
/// If there are already messages in the folder queue, those will be processed first...
/// </summary>
/// <returns></returns>
async Task IWork.StartAsync()
{
// first enumerate top level and add to queue.
if (opts.FileOnlyMode)
{
Log.Always("FILE_RUNNER_START");
folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + opts.WorkerId);
fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + opts.WorkerId);
await StartFileRunner().ConfigureAwait(false);
}
else if (opts.LargeFileOnlyMode)
{
Log.Always("LARGE_FILE_RUNNER_START");
await StartLargeFileRunner().ConfigureAwait(false);
}
else
{
if (!opts.Resume)
{
SubmitBatchedTopLevelWorkitems(opts);
}
// ToDo: Add Job / Queue Id to log events
Log.Always(FixedStrings.StartingFolderQueueLogJson + "\":\"" + opts.WorkerId);
await ProcessWorkQueue(folderCopyQueue, false).ConfigureAwait(true);
}
}
protected CopyJob(CopierOptions opts)
{
// Folder WorkItem mgmt needs late init, as we don't need more queues than folders!
largeFileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.LargeFilesQueueName);
WorkItemSubmissionController = WorkItemMgmtFactory.CreateAzureWorkItemSubmissionController(opts.WorkerCount, opts.WorkerId);
folderDoneSet = AzureServiceFactory.GetFolderDoneSet();
originalWorkerId = opts.WorkerId;
}
private async void UpdateFolderStats(int Id)
{
var folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + Id.ToString());
var sizeFolderQueue = await GetQueueSize(folderCopyQueue).ConfigureAwait(true);
if (sizeFolderQueue > 0)
{
totalFolderMessages += sizeFolderQueue;
if (sizeFolderQueue > largestFolderQueue)
{
largestFolderQueue = sizeFolderQueue;
}
}
string eval = EvalString(sizeFolderQueue);
cw.WriteAt(eval, Id, FolderStatsRowOffset);
}
private async void UpdateLargeFileStats()
{
var largeFileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.LargeFilesQueueName);
var sizeLargeFilesQueue = await GetQueueSize(largeFileCopyQueue).ConfigureAwait(false);
if (sizeLargeFilesQueue > 0)
{
totalFileMessages += sizeLargeFilesQueue;
if (sizeLargeFilesQueue > largeFilesQueueLength)
{
largestFolderQueue = sizeLargeFilesQueue;
}
}
string eval = EvalString(sizeLargeFilesQueue);
cw.WriteAt(blanks, LargeFileStatsColOffset, LargeFileStatsRowOffset);
cw.WriteAt(eval, LargeFileStatsColOffset, LargeFileStatsRowOffset);
}
private async void UpdateFileStats(int Id)
{
// ToDo: we need to reduce the number of object creations... use an iterable collection to store all queues
// questions around efficiency and connection limits... ?
var folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + Id.ToString());
var sizeFileQueue = await GetQueueSize(folderCopyQueue).ConfigureAwait(false);
if (sizeFileQueue > 0)
{
totalFileMessages += sizeFileQueue;
if (sizeFileQueue > largestFileQueue)
{
largestFileQueue = sizeFileQueue;
}
}
string eval = EvalString(sizeFileQueue);
cw.WriteAt(eval, Id, FileStatsRowOffset);
// ToDo: Work out a way to implement a more accurate display using the below...
//cw.WriteAt(sizeFileQueue.ToString(), Id + CurrentStattColumnOffset, FileStatsRowOffset);
//CurrentStattColumnOffset += sizeFileQueue.ToString().Length + 1;
}
