public DownloadResults Download()
{
TickableProgressTick tickableProgress = new TickableProgressTick(_progressHost, _urlsToDownload.Count);
Hashtable workItems = new Hashtable();
foreach (string url in _urlsToDownload.Keys)
{
DownloadWorkItem workItem = new DownloadWorkItem(url, (int)_urlsToDownload[url], tickableProgress);
workItems.Add(url, workItem);
_downloadQueue.Enqueue(workItem);
}
ParallelExecution execution = new
ParallelExecution(new ThreadStart(DoWork), _threadCount);
execution.Execute();
DownloadResults results = new DownloadResults();
foreach (string url in workItems.Keys)
{
results.AddResult(url, ((DownloadWorkItem)workItems[url]).FilePath);
}
return(results);
}
public ResultMessageEntity VerifyGetAuthenticationDetail(AuthenticationReqMsgEntity authenticationMsgEntity)
{
IParallelExecution <AuthenticationReqMsgEntity> parallelExecution = new ParallelExecution <AuthenticationReqMsgEntity>();
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = IsPartnerValid, IsParallel = true, RequestObject = authenticationMsgEntity
});
return(parallelExecution.Execute());
}
public ResultMessageEntity ValidCreateUser(CreateUserMessageEntity createUserMessageEntity)
{
IParallelExecution <CreateUserMessageEntity> parallelExecution = new ParallelExecution <CreateUserMessageEntity>();
parallelExecution.Add(new ParallelAction <CreateUserMessageEntity> {
Activity = IsCreateUserMessageEntityValid, RequestObject = createUserMessageEntity
});
return(parallelExecution.Execute());
}
public ResultMessageEntity ProcessorGetAuthenticationDetail(AuthenticationReqMsgEntity authenticationMsgEntity)
{
IParallelExecution <AuthenticationReqMsgEntity> parallelExecution = new ParallelExecution <AuthenticationReqMsgEntity>();
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = GenerateToken, IsParallel = true, RequestObject = authenticationMsgEntity
});
return(parallelExecution.Execute());
}
public static void Explain(this ParallelExecution runnable, TextWriter writer)
{
writer.WriteLine(@"
- Nature of Task API allows to combine concurrency and explicit parallelism.
- Degree of Parallelism = Number of Threads used from worker pool.
- Async all the way: Try to avoid blocking code in async body if you can
- In some scenarios it is OK to call blocking IO bound code in async body
- Top level caller can always offload if required
");
}
public void ParallelExecution_BuildWithNonGenericProcessorImplementation_ItemsArgumentIsNullException()
{
//Arrange
List <string> items = null;
Func <string, Task> mockProcessor = item => Task.CompletedTask;
//Assert
Assert.Throws <ArgumentNullException>(() => ParallelExecution.Build(items, mockProcessor));
}
public void ParallelExecution_BuildWithNonGenericProcessorImplementation_ProviderArgumentIsNullException()
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > mockProvider = null;
Func <string, Task> mockProcessor = item => Task.CompletedTask;
//Assert
Assert.Throws <ArgumentNullException>(() => ParallelExecution.Build(mockProvider, mockProcessor));
}
public async Task ParallelExecutionImplementation_RunAsync_ProviderReturnedNullObject()
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > provider =
skip => Task.FromResult <IEnumerable <string> >(null);
Func <string, Task> processor = item => Task.CompletedTask;
//Assert
await Assert.ThrowsAsync <ArgumentNullException>(
() => ParallelExecution.Build(provider, processor).RunAsync());
}
public void ParallelExecution_BuildWithNonGenericProcessorImplementation_ItemsAsSource_Success()
{
//Arrange
var items = _fixture.CreateMany <string>(50).ToList();
Func <string, Task> mockProcessor = item => Task.CompletedTask;
//Act
var sut = ParallelExecution.Build(items, mockProcessor);
//Assert
Assert.NotNull(sut);
Assert.True(sut.GetType().GetGenericTypeDefinition() == typeof(ParallelExecutionImplementation <>));
}
public async Task ParallelExecutionImplementation_RunAsync_ItemsAsSource_Success()
{
//Arrange
var items = _fixture.CreateMany <string>(50).ToList();
Func <string, Task> processor = item => Task.CompletedTask;
//Act
await ParallelExecution.Build(items, processor).RunAsync().ContinueWith(run =>
{
//Assert
Assert.True(run.IsCompletedSuccessfully);
});
}
public async Task ParallelExecutionImplementation_RunAsync_GenericTaskReturnType_ProcessorThrowsExeception()
{
//Arrange
var numOfItemsInRepository = 50;
var repositoryMock = _fixture.CreateMany <string>(numOfItemsInRepository).ToList();
Func <int, Task <IEnumerable <string> > > provider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(2));
Func <string, Task <int> > processor = item => throw new Exception();
//Assert
await Assert.ThrowsAsync <Exception>(() => ParallelExecution.Build(provider, processor).RunAsync());
}
public void ParallelExecution_BuildWithNonGenericProcessorImplementation_Success()
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > mockProvider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(2));
Func <string, Task> mockProcessor = item => Task.CompletedTask;
//Act
var sut = ParallelExecution.Build(mockProvider, mockProcessor);
//Assert
Assert.NotNull(sut);
Assert.True(sut.GetType().GetGenericTypeDefinition() == typeof(ParallelExecutionImplementation <>));
}
public async Task ParallelExecutionImplementation_RunAsync_CancellationRequested()
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > provider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(1));
Func <string, Task> processor = item => Task.Delay(50);
var cts = new CancellationTokenSource();
cts.CancelAfter(1);
//Assert
await Assert.ThrowsAsync <OperationCanceledException>(
() => ParallelExecution.Build(provider, processor).RunAsync(cts.Token));
}
public void ParallelExecution_BuildWithNonGenericProcessorImplementation_MaxConcurrentThreadsArgumentIsEqualOrLowerThanZeroException(
int maxConcurrentThreadsValue)
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > mockProvider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(2));
Func <string, Task> mockProcessor = item => Task.CompletedTask;
//Assert
Assert.Throws <ArgumentException>(
() => ParallelExecution.Build(
mockProvider,
mockProcessor,
maxConcurrentThreadsValue));
}
public ResultMessageEntity ValidGetAuthenticationDetail(AuthenticationReqMsgEntity authenticationMsgEntity)
{
IParallelExecution <AuthenticationReqMsgEntity> parallelExecution = new ParallelExecution <AuthenticationReqMsgEntity>();
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = IsAuthenticationReqMsgEntityValid, RequestObject = authenticationMsgEntity
});
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = IsUserNameValid, RequestObject = authenticationMsgEntity, IsParallel = true
});
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = IsPasswordValid, RequestObject = authenticationMsgEntity, IsParallel = true
});
parallelExecution.Add(new ParallelAction <AuthenticationReqMsgEntity> {
Activity = IsPartnerKeyValid, RequestObject = authenticationMsgEntity, IsParallel = true
});
return(parallelExecution.Execute());
}
public async Task ParallelExecutionImplementation_RunAsync_Success()
{
//Arrange
var repositoryMock = _fixture.CreateMany <string>(50).ToList();
Func <int, Task <IEnumerable <string> > > provider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(2));
Func <string, Task> processor = item => Task.CompletedTask;
//Act
await ParallelExecution.Build(provider, processor).RunAsync()
.ContinueWith(run =>
{
//Assert
Assert.True(run.IsCompletedSuccessfully);
});
}
public async Task ParallelExecutionImplementation_RunAsync_GenericTaskReturnType_Success()
{
//Arrange
var numOfItemsInRepository = 50;
var repositoryMock = _fixture.CreateMany <string>(numOfItemsInRepository).ToList();
Func <int, Task <IEnumerable <string> > > provider =
skip => Task.FromResult(repositoryMock.Skip(skip).Take(2));
var processorReturnedValue = 5;
Func <string, Task <int> > processor = item => Task.FromResult(processorReturnedValue);
//Act
var results = await ParallelExecution.Build(provider, processor).RunAsync();
//Assert
Assert.NotNull(results);
Assert.True(results.Count() == numOfItemsInRepository);
Assert.True(results.All(x => x == processorReturnedValue));
}
public DownloadResults Download()
{
TickableProgressTick tickableProgress = new TickableProgressTick(_progressHost, _urlsToDownload.Count);
Hashtable workItems = new Hashtable();
foreach (string url in _urlsToDownload.Keys)
{
DownloadWorkItem workItem = new DownloadWorkItem(url, (int)_urlsToDownload[url], tickableProgress);
workItems.Add(url, workItem);
_downloadQueue.Enqueue(workItem);
}
ParallelExecution execution = new
ParallelExecution(new ThreadStart(DoWork), _threadCount);
execution.Execute();
DownloadResults results = new DownloadResults();
foreach (string url in workItems.Keys)
{
results.AddResult(url, ((DownloadWorkItem)workItems[url]).FilePath);
}
return results;
}
/// <summary>
/// Downloads the pages and their references, providing progress feedback
/// </summary>
/// <param name="progressHost">The progresshost to use for feedback</param>
/// <returns>this</returns>
public object Download(IProgressHost progressHost)
{
// Prepare the list of references to download
progressHost.UpdateProgress(Res.Get(StringId.ProgressPreparingListOfFiles));
foreach (PageToDownload pageToDownload in _pagesToDownload)
{
// Lay down a placeholder file with the correct file name
try
{
string destination = Path.Combine(_siteStorage.BasePath, pageToDownload.RelativePath);
destination = PathHelper.GetNonConflictingPath(destination);
pageToDownload.FileName = Path.GetFileName(destination);
using (Stream htmlStream = _siteStorage.Open(destination, AccessMode.Write)) { }
}
catch (Exception e)
{
HandleException(e);
}
foreach (ReferenceToDownload reference in pageToDownload.References)
{
// Don't add the same item more than once
if (!_referencesToDownload.ContainsKey(reference.AbsoluteUrl))
{
_referencesToDownload.Add(reference.AbsoluteUrl, reference);
}
}
}
// Enqueue the work items
progressHost.UpdateProgress(Res.Get(StringId.ProgressStartingDownloadOfReferences));
IProgressHost[] progressHosts = new JointProgressHosts(progressHost, _referencesToDownload.Count, 8000, 10000).ProgressHosts;
int tickNum = 0;
foreach (ReferenceToDownload reference in _referencesToDownload.Values)
{
workQueue.Enqueue(new DownloadWorkItem(reference, _siteStorage, progressHosts[tickNum++]));
}
// Start up the parallel execution of the downloads
ParallelExecution parallelExecution = new ParallelExecution(new ThreadStart(WorkerThreadStart), 2);
parallelExecution.Execute();
parallelExecution = null;
// Now go through and get HTML for each page, and emit the HTML to disk
ProgressTick allPagesProgress = new ProgressTick(progressHost, 2000, 10000);
for (int i = 0; i < _pagesToDownload.Length; i++)
{
try
{
allPagesProgress.UpdateProgress(i, _pagesToDownload.Length, string.Format(CultureInfo.CurrentCulture, Res.Get(StringId.ProgressSaving), _pagesToDownload[i].FileName));
WriteHtmlToDisk(_pagesToDownload[i], _siteStorage);
}
catch (Exception e)
{
HandleException(e);
}
if (allPagesProgress.CancelRequested)
{
throw new OperationCancelledException();
}
}
// We're complete!
progressHost.UpdateProgress(1, 1, Res.Get(StringId.ProgressDownloadFinished));
return(this);
}
public static void PrintStart(this ParallelExecution runnable, int element)
{
Console.WriteLine($"start {element} / {Thread.CurrentThread.ManagedThreadId}");
}
public static void PrintEnd(this ParallelExecution runnable, Int32 element)
{
Console.WriteLine($"done {element} / {Thread.CurrentThread.ManagedThreadId}");
}
/// <summary>
/// Initializes with prescending async operation
/// execution of which is used as trigger of other operations.
/// </summary>
/// <param name="executionContext">Precending execution context.</param>
public ExecutionType(IExecutionContext executionContext)
{
InParallel = new ParallelExecution(executionContext);
OneByOne = new SequentialExecution(executionContext);
}
/// <summary>
/// Main function.
/// </summary>
/// <param name="args">The args.</param>
static int Main(
string[] args)
{
ExitCode = 0;
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(CurrentDomain_UnhandledException);
try
{
LoadAllConfigurationKeys();
currentConfiguration = DataHelpers.usp_PullNextParallelExecution(null);
if (currentConfiguration != null)
{
try
{
// Max Number of cores...
int nbCores = (DataHelpers.MaxDegreeOfParallelism ?? 8);
if ((currentConfiguration.MaxDegreeOfParallelism < -1) ||
(currentConfiguration.MaxDegreeOfParallelism == 0) ||
(currentConfiguration.MaxDegreeOfParallelism > nbCores))
{
currentConfiguration.MaxDegreeOfParallelism = nbCores;
}
List <Partition> partitions = DataHelpers.GetPartitions(
null,
currentConfiguration.PartitionStatement);
foreach (Partition partition in partitions)
{
partition.SessionId = currentConfiguration.SessionId;
partition.PartitionId = DataHelpers.usp_CreateParallelExecutionPartition(
null,
currentConfiguration.SessionId);
foreach (KeyValuePair <int, object> parameter in partition.Parameters)
{
DataHelpers.usp_CreateParallelExecutionPartitionParameter(
null,
currentConfiguration.SessionId,
partition.PartitionId,
parameter.Key,
parameter.Value);
}
}
ParallelLoopResult ret = Parallel.ForEach(
partitions,
new ParallelOptions()
{
MaxDegreeOfParallelism = currentConfiguration.MaxDegreeOfParallelism
},
(partition, state) =>
{
bool success = false;
string error = null;
try
{
DataHelpers.usp_SetStatus_ParallelExecutionPartition(
null,
currentConfiguration.SessionId,
partition.PartitionId,
SessionPartitionStatus.Processing,
null);
using (SqlConnection connection = new SqlConnection(DataHelpers.MainConnectionString))
{
connection.Open();
connection.FireInfoMessageEventOnUserErrors = false;
connection.InfoMessage += (sender, message) =>
{
DataHelpers.usp_LogParallelExecutionEvent(
null,
currentConfiguration.SessionId,
partition.PartitionId,
TranslateSeverity(message.Errors),
"InfoMessage",
message.Message);
};
DataHelpers.Execute_Statement_NonQuery(
connection,
currentConfiguration.PartitionCommand,
partition.GetSqlParameters());
}
success = true;
}
catch (SqlException se)
{
error = se.Message;
DataHelpers.usp_LogParallelExecutionEvent(
null,
currentConfiguration.SessionId,
partition.PartitionId,
TranslateSeverity(se.Errors),
se.Message,
se.ToString());
}
catch (Exception e)
{
error = e.Message;
DataHelpers.usp_LogParallelExecutionEvent(
null,
currentConfiguration.SessionId,
partition.PartitionId,
ParallelExecutionEventStatus.Error,
e.Message,
e.ToString());
}
finally
{
partition.PartitionStatus = (success ? SessionPartitionStatus.Complete : SessionPartitionStatus.Failed);
DataHelpers.usp_SetStatus_ParallelExecutionPartition(
null,
currentConfiguration.SessionId,
partition.PartitionId,
(success ? SessionPartitionStatus.Complete : SessionPartitionStatus.Failed),
error);
}
if (!success &&
!currentConfiguration.ContinueOnError)
{
DataHelpers.usp_LogParallelExecutionEvent(
null,
currentConfiguration.SessionId,
partition.PartitionId,
ParallelExecutionEventStatus.Important,
"Stopping",
"Partition processing failed and the session is configured to stop on first partition failure (ContinueOnError = false).");
state.Stop();
}
});
if (!ret.IsCompleted)
{
currentConfiguration.SessionStatus = SessionPartitionStatus.Failed;
}
else
{
if (partitions.Where(t => t.PartitionStatus != SessionPartitionStatus.Complete).Any())
{
// In theory, we should never end up there, but...
currentConfiguration.SessionStatus = SessionPartitionStatus.Failed;
}
else
{
currentConfiguration.SessionStatus = SessionPartitionStatus.Complete;
}
}
}
finally
{
// Make sure we are setting the status to either Failed or Complete...
if (currentConfiguration.SessionStatus < SessionPartitionStatus.Failed)
{
currentConfiguration.SessionStatus = SessionPartitionStatus.Failed;
}
// Update overall status...
DataHelpers.usp_SetStatus_ParallelExecution(
null,
currentConfiguration.SessionId,
currentConfiguration.SessionStatus,
null);
}
}
}
catch (Exception e)
{
TryAndReportException(e);
}
return(ExitCode);
}
/// <summary>
/// Initializes with prescending async operation
/// execution of which is used as trigger of other operations.
/// </summary>
/// <param name="executionContext">Precending execution context.</param>
public ExecutionType(IExecutionContext executionContext)
{
InParallel = new ParallelExecution(executionContext);
OneByOne = new SequentialExecution(executionContext);
}