public IEnumerable <BatchedWorkItem> CreateWorkItems(IWorkBatchFactory workBatchFactory, CancellationToken cancellationToken) { var blobs = BlobHelper.GetAllFilesInContainerWithPath(string.Empty); var localFile = Directory.GetFiles(".", "*", SearchOption.AllDirectories); var toDownload = blobs.Where(blob => NotDowloaded(blob, localFile)).ToList(); return(GetWorkItems(toDownload, workBatchFactory)); }
public IEnumerable <BatchedWorkItem> CreateWorkItems(IWorkBatchFactory workBatchFactory, CancellationToken cancellationToken) { FormattedConsole.WriteLine($"Producer {ProducerId} - Checking for work..."); while (_workAvailableRepo.WorkAvailable()) { foreach (var workItem in GetWorkItems(workBatchFactory)) { yield return(workItem); } } }
private IEnumerable <BatchedWorkItem> GetWorkItems(IWorkBatchFactory workBatchFactory, bool startup = false) { var workBatch = workBatchFactory.Create(3, new DummyWorkBatch(), startup: startup); var workItems = new List <BatchedWorkItem>(); for (var i = 0; i < 3; i++) { workItems.Add(new BatchedWorkItem(workBatch, new DummyWorkItem(_rand.Next(0, 100).ToString()))); } FormattedConsole.WriteLine($"Producer {ProducerId} has created {workItems[0].Id}, {workItems[1].Id}, {workItems[2].Id}"); return(workItems); }
private static IEnumerable <BatchedWorkItem> GetWorkItems(IReadOnlyCollection <Uri> fileLocations, IWorkBatchFactory workBatchFactory, string batchId = null) { var batch = workBatchFactory.Create(fileLocations.Count, new BlobWatcherWorkBatch(batchId, ProcessResults)); FormattedConsole.WriteLine($"Created batch {batch.Id} with {fileLocations.Count} blobs"); return(fileLocations .Select(uri => new BatchedWorkItem(batch, new BlobWatcherWorkItem(uri.AbsoluteUri, batch)))); }
public IEnumerable <BatchedWorkItem> CreateStartupWorkItems(IWorkBatchFactory workBatchFactory, CancellationToken cancellationToken) { // Load everything at startup return(GetWorkItems(BlobHelper.GetAllFilesInContainerWithPath("*"), workBatchFactory)); }
public IEnumerable <BatchedWorkItem> CreateStartupWorkItems(IWorkBatchFactory workBatchFactory, CancellationToken cancellationToken) { FormattedConsole.WriteLine($"Producer {ProducerId} - Startup"); return(GetWorkItems(workBatchFactory, true)); }