private void MoveWorkToNextQueue() { opts.WorkerId++; if (opts.WorkerId >= opts.WorkerCount) { opts.WorkerId = 0; } int batchIndex = GetBatchStartingIndex(topLevelFoldersCount, opts); Log.Always("BATCH INDEX " + batchIndex); Log.Always(FixedStrings.ProcessingQueue + batchIndex); if (topLevelFoldersCount > opts.WorkerCount) { // We have more folders than workers, we assign queues based on ThreadId folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + opts.WorkerId); fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + opts.WorkerId); } else { // We have more workers than folders, we assign queues based on zero based folder index folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + batchIndex); fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + batchIndex); } }
/// <summary> /// Starts the processing of work given the path provided to the tool /// If there are already messages in the folder queue, those will be processed first... /// </summary> /// <returns></returns> async Task IWork.StartAsync() { // first enumerate top level and add to queue. if (opts.FileOnlyMode) { Log.Always("FILE_RUNNER_START"); folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + opts.WorkerId); fileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + opts.WorkerId); await StartFileRunner().ConfigureAwait(false); } else if (opts.LargeFileOnlyMode) { Log.Always("LARGE_FILE_RUNNER_START"); await StartLargeFileRunner().ConfigureAwait(false); } else { if (!opts.Resume) { SubmitBatchedTopLevelWorkitems(opts); } // ToDo: Add Job / Queue Id to log events Log.Always(FixedStrings.StartingFolderQueueLogJson + "\":\"" + opts.WorkerId); await ProcessWorkQueue(folderCopyQueue, false).ConfigureAwait(true); } }
protected CopyJob(CopierOptions opts) { // Folder WorkItem mgmt needs late init, as we don't need more queues than folders! largeFileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.LargeFilesQueueName); WorkItemSubmissionController = WorkItemMgmtFactory.CreateAzureWorkItemSubmissionController(opts.WorkerCount, opts.WorkerId); folderDoneSet = AzureServiceFactory.GetFolderDoneSet(); originalWorkerId = opts.WorkerId; }
private async void UpdateFolderStats(int Id) { var folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFolderQueueName + Id.ToString()); var sizeFolderQueue = await GetQueueSize(folderCopyQueue).ConfigureAwait(true); if (sizeFolderQueue > 0) { totalFolderMessages += sizeFolderQueue; if (sizeFolderQueue > largestFolderQueue) { largestFolderQueue = sizeFolderQueue; } } string eval = EvalString(sizeFolderQueue); cw.WriteAt(eval, Id, FolderStatsRowOffset); }
private async void UpdateLargeFileStats() { var largeFileCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.LargeFilesQueueName); var sizeLargeFilesQueue = await GetQueueSize(largeFileCopyQueue).ConfigureAwait(false); if (sizeLargeFilesQueue > 0) { totalFileMessages += sizeLargeFilesQueue; if (sizeLargeFilesQueue > largeFilesQueueLength) { largestFolderQueue = sizeLargeFilesQueue; } } string eval = EvalString(sizeLargeFilesQueue); cw.WriteAt(blanks, LargeFileStatsColOffset, LargeFileStatsRowOffset); cw.WriteAt(eval, LargeFileStatsColOffset, LargeFileStatsRowOffset); }
private async void UpdateFileStats(int Id) { // ToDo: we need to reduce the number of object creations... use an iterable collection to store all queues // questions around efficiency and connection limits... ? var folderCopyQueue = WorkItemMgmtFactory.CreateAzureWorkItemMgmt(CloudObjectNameStrings.CopyFilesQueueName + Id.ToString()); var sizeFileQueue = await GetQueueSize(folderCopyQueue).ConfigureAwait(false); if (sizeFileQueue > 0) { totalFileMessages += sizeFileQueue; if (sizeFileQueue > largestFileQueue) { largestFileQueue = sizeFileQueue; } } string eval = EvalString(sizeFileQueue); cw.WriteAt(eval, Id, FileStatsRowOffset); // ToDo: Work out a way to implement a more accurate display using the below... //cw.WriteAt(sizeFileQueue.ToString(), Id + CurrentStattColumnOffset, FileStatsRowOffset); //CurrentStattColumnOffset += sizeFileQueue.ToString().Length + 1; }