protected override WorkStatus DoAlways(CancellationToken cancellationToken)
{
CloudConsole.WriteLine($"Consumer {Thread.CurrentThread.ManagedThreadId} consuming {Id}");
cancellationToken.WaitHandle.WaitOne(2000);
return(WorkStatus.Successful);
}
private static IWorkItem[] GetWorkItems(string sourcePath, Func <Uri, WorkBatch, IWorkItem> workItemFactoryMethod, int batchId)
{
var fileLocations = GetAllFilesWithPatternInBlob(sourcePath, ".*part.*");
var batch = new WorkBatch(fileLocations.Length, batchId.ToString());
CloudConsole.WriteLine($"Created batch {batch.Id} with {fileLocations.Length} blobs");
return(fileLocations
.Select(uri => workItemFactoryMethod(uri, batch))
.ToArray());
}
protected override void DoFinally(CancellationToken cancellationToken)
{
var results = new List <string>();
foreach (var done in Batch.CompletedWorkItems)
{
results.Add((string)done.Result);
}
_storeResults(results.ToArray());
CloudConsole.WriteLine($"Batch {Batch.Id} done");
}
private static IWorkItem[] GetWorkItems(CancellationToken cancellationToken)
{
var rand = new Random();
var workBatch = new WorkBatch(3);
var workItems = new[]
{
new DummyWorkItem(rand.Next(), workBatch, cancellationToken),
new DummyWorkItem(rand.Next(), workBatch, cancellationToken),
new DummyWorkItem(rand.Next(), workBatch, cancellationToken)
};
CloudConsole.WriteLine(
$"Producer {Thread.CurrentThread.ManagedThreadId} has created " +
$"{workItems[0].Id}, {workItems[1].Id}, {workItems[2].Id}");
return(workItems.Cast <IWorkItem>().ToArray());
}
protected override void DoAlways(CancellationToken cancellationToken)
{
try
{
var storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["Azure.Storage.ConnectionString"]);
var blockBlob = GetBlobReference(storageAccount, Id);
Result = DownloadBlobFile(blockBlob);
WorkItemStatus = WorkItemStatus.Successful;
}
catch (Exception e)
{
CloudConsole.WriteLine($"Work Item {Id} of {Batch.Id} failed, exception {e.Message}");
WorkItemStatus = WorkItemStatus.Failed;
}
}
private static IWorkItem[] GetWorkItems(EventWaitHandle started = null)
{
var rand = new Random();
var workBatch = new WorkBatch(3, new DummyWorkBatch(started));
var workItems = new[]
{
new BatchedWorkItem(workBatch, new DummyWorkItem(rand.Next())),
new BatchedWorkItem(workBatch, new DummyWorkItem(rand.Next())),
new BatchedWorkItem(workBatch, new DummyWorkItem(rand.Next()))
};
CloudConsole.WriteLine(
$"Producer {Thread.CurrentThread.ManagedThreadId} has created " +
$"{workItems[0].Id}, {workItems[1].Id}, {workItems[2].Id}");
return(workItems.Cast <IWorkItem>().ToArray());
}
private static void Main(string[] args)
{
ServicePointManager.DefaultConnectionLimit = 1000;
var tokenSource = new CancellationTokenSource();
WorkerPoolExample.StartPool(
false,
new TimeSpan(0, 0, 0, 30),
new TimeSpan(0, 0, 0, 10),
100,
tokenSource.Token);
CloudConsole.WriteLine("Pool started correctly!");
while (Console.ReadKey().KeyChar != 'q')
{
}
tokenSource.Cancel();
}
public IWorkItem[] Create(CancellationToken cancellationToken)
{
GC.Collect();
if (_checkBoundedBufferStatusFunc())
{
CloudConsole.WriteLine("Bounded buffer healthy");
}
var path = ConfigurationManager.AppSettings["Azure.Storage.Path"];
if (!_workAvailableRepo.IsWorkAvailable())
{
return(null);
}
return(GetWorkItems(
path,
(uri, batch) =>
new ReadFromBlobWorkItem(StoreResults, uri.AbsoluteUri, batch, cancellationToken)
as IWorkItem,
cancellationToken));
}
protected override void DoFinally(CancellationToken cancellationToken)
{
try
{
CloudConsole.WriteLine($"{Batch.Id} dofinally started");
var results = new List <byte>();
foreach (var result in Batch.Results)
{
results.AddRange((byte[])result);
}
_storeResults(results.ToArray(), int.Parse(Batch.Id));
//// This is important when using large object heap extensively
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
CloudConsole.WriteLine($"Batch {Batch.Id} done");
}
catch (Exception e)
{
}
}
public IWorkItem[] Startup(CancellationToken cancellationToken, ManualResetEvent started = null)
{
CloudConsole.WriteLine("Producer startup");
return(GetWorkItems(cancellationToken));
}
protected override void DoFinally(CancellationToken cancellationToken)
{
CloudConsole.WriteLine($"Consumer {Thread.CurrentThread.ManagedThreadId} consuming {Id}, LAST OF THE BATCH");
}
protected override void DoAlways(CancellationToken cancellationToken)
{
CloudConsole.WriteLine($"Consumer {Thread.CurrentThread.ManagedThreadId} consuming {Id}");
cancellationToken.WaitHandle.WaitOne(2000);
}
