public ProcessingQueue(Action <Exception>?exceptionLogger, string?exceptionHeader = null)
{
this.exceptionHeader = exceptionHeader ?? "error while running queued task";
this.logException = exceptionLogger ?? Console.Error.WriteLine;
this.scheduler = new ConcurrentExclusiveSchedulerPair();
}
public Form1()
{
InitializeComponent();
// Create an ActionBlock<CheckBox> object that toggles the state
// of CheckBox objects.
// Specifying the current synchronization context enables the
// action to run on the user-interface thread.
var toggleCheckBox = new ActionBlock <CheckBox>(checkBox =>
{
checkBox.Checked = !checkBox.Checked;
},
new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext()
});
// Create a ConcurrentExclusiveSchedulerPair object.
// Readers will run on the concurrent part of the scheduler pair.
// The writer will run on the exclusive part of the scheduler pair.
var taskSchedulerPair = new ConcurrentExclusiveSchedulerPair();
// Create an ActionBlock<int> object for each reader CheckBox object.
// Each ActionBlock<int> object represents an action that can read
// from a resource in parallel to other readers.
// Specifying the concurrent part of the scheduler pair enables the
// reader to run in parallel to other actions that are managed by
// that scheduler.
var readerActions =
from checkBox in new[] { checkBox1, checkBox2, checkBox3 }
select new ActionBlock <int>(milliseconds =>
{
// Toggle the check box to the checked state.
toggleCheckBox.Post(checkBox);
// Perform the read action. For demonstration, suspend the current
// thread to simulate a lengthy read operation.
Thread.Sleep(milliseconds);
// Toggle the check box to the unchecked state.
toggleCheckBox.Post(checkBox);
},
new ExecutionDataflowBlockOptions
{
TaskScheduler = taskSchedulerPair.ConcurrentScheduler
});
// Create an ActionBlock<int> object for the writer CheckBox object.
// This ActionBlock<int> object represents an action that writes to
// a resource, but cannot run in parallel to readers.
// Specifying the exclusive part of the scheduler pair enables the
// writer to run in exclusively with respect to other actions that are
// managed by the scheduler pair.
var writerAction = new ActionBlock <int>(milliseconds =>
{
// Toggle the check box to the checked state.
toggleCheckBox.Post(checkBox4);
// Perform the write action. For demonstration, suspend the current
// thread to simulate a lengthy write operation.
Thread.Sleep(milliseconds);
// Toggle the check box to the unchecked state.
toggleCheckBox.Post(checkBox4);
},
new ExecutionDataflowBlockOptions
{
TaskScheduler = taskSchedulerPair.ExclusiveScheduler
});
// Link the broadcaster to each reader and writer block.
// The BroadcastBlock<T> class propagates values that it
// receives to all connected targets.
foreach (var readerAction in readerActions)
{
mBroadcaster.LinkTo(readerAction);
}
mBroadcaster.LinkTo(writerAction);
// Start the timer.
timer1.Start();
}
public static void TestSchedulerNesting()
{
// Create a hierarchical set of scheduler pairs
var cespParent = new ConcurrentExclusiveSchedulerPair();
var cespChild1 = new ConcurrentExclusiveSchedulerPair(cespParent.ConcurrentScheduler);
var cespChild1Child1 = new ConcurrentExclusiveSchedulerPair(cespChild1.ConcurrentScheduler);
var cespChild1Child2 = new ConcurrentExclusiveSchedulerPair(cespChild1.ExclusiveScheduler);
var cespChild2 = new ConcurrentExclusiveSchedulerPair(cespParent.ExclusiveScheduler);
var cespChild2Child1 = new ConcurrentExclusiveSchedulerPair(cespChild2.ConcurrentScheduler);
var cespChild2Child2 = new ConcurrentExclusiveSchedulerPair(cespChild2.ExclusiveScheduler);
// these are ordered such that we will complete the child schedulers before we complete their parents. That way
// we don't complete a parent that's still in use.
var cesps = new[] {
cespChild1Child1,
cespChild1Child2,
cespChild1,
cespChild2Child1,
cespChild2Child2,
cespChild2,
cespParent,
};
// Get the schedulers from all of the pairs
List <TaskScheduler> schedulers = new List <TaskScheduler>();
foreach (var s in cesps)
{
schedulers.Add(s.ConcurrentScheduler);
schedulers.Add(s.ExclusiveScheduler);
}
// Keep track of all created tasks
var tasks = new List <Task>();
// Queue lots of work to each scheduler
foreach (var scheduler in schedulers)
{
// Create a function that schedules and inlines recursively queued tasks
Action <int> recursiveWork = null;
recursiveWork = depth =>
{
if (depth > 0)
{
Action work = () =>
{
var sw = new SpinWait();
while (!sw.NextSpinWillYield)
{
sw.SpinOnce();
}
recursiveWork(depth - 1);
};
TaskFactory factory = new TaskFactory(scheduler);
Debug.WriteLine(string.Format("Start tasks in scheduler {0}", scheduler.Id));
Task t1 = factory.StartNew(work); Task t2 = factory.StartNew(work); Task t3 = factory.StartNew(work);
Task.WaitAll(t1, t2, t3);
}
};
for (int i = 0; i < 2; i++)
{
tasks.Add(Task.Factory.StartNew(() => recursiveWork(2), CancellationToken.None, TaskCreationOptions.None, scheduler));
}
}
// Wait for all tasks to complete, then complete the schedulers
Task.WaitAll(tasks.ToArray());
foreach (var cesp in cesps)
{
cesp.Complete();
cesp.Completion.Wait();
}
}
void Test2()
{
var schedulerPair = new ConcurrentExclusiveSchedulerPair();
TaskScheduler concurrent = schedulerPair.ConcurrentScheduler;
TaskScheduler exclusive = schedulerPair.ExclusiveScheduler;
}
public static void TaskContinuation()
{
int taskCount = Environment.ProcessorCount;
int maxDOP = Int32.MaxValue;
int maxNumberExecutionsPerTask = 1;
int data = 0;
Task[] allTasks = new Task[taskCount + 1];
CancellationTokenSource[] cts = new CancellationTokenSource[taskCount + 1];
for (int i = 0; i <= taskCount; i++)
{
cts[i] = new CancellationTokenSource();
}
CancellationTokenSource cts2 = new CancellationTokenSource();
ConcurrentExclusiveSchedulerPair scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, maxDOP, maxNumberExecutionsPerTask);
for (int i = 0; i <= taskCount; i++)
{
int j = i;
allTasks[i] = new Task(() =>
{
new TaskFactory(TaskScheduler.Current).StartNew(() => { }).
ContinueWith((task, o) =>
{
int d = (int)o;
Interlocked.Add(ref data, d);
}, j).
ContinueWith((task, o) =>
{
int d = (int)o;
Interlocked.Add(ref data, d);
cts[d].Cancel();
if (d <= taskCount)
{
throw new OperationCanceledException(cts[d].Token);
}
return("Done");
}, j, cts[j].Token).
ContinueWith((task, o) =>
{
int d = (int)o;
Interlocked.Add(ref data, d);
}, j, CancellationToken.None, TaskContinuationOptions.OnlyOnCanceled, TaskScheduler.Default).Wait(Int32.MaxValue - 1, cts2.Token);
});
allTasks[i].Start(scheduler.ConcurrentScheduler);
}
Task.WaitAll(allTasks, int.MaxValue - 1, CancellationToken.None);
Debug.WriteLine("Tasks ended: result {0}", data);
Task completion = scheduler.Completion;
scheduler.Complete();
completion.Wait();
int expectedResult = 3 * taskCount * (taskCount + 1) / 2;
Assert.Equal(expectedResult, data);
Assert.NotEqual(TaskScheduler.Default.Id, scheduler.ConcurrentScheduler.Id);
Assert.NotEqual(TaskScheduler.Default.Id, scheduler.ExclusiveScheduler.Id);
}
public static void GetTaskSchedulersForDebugger_DebuggerAttached_ReturnsAllSchedulers()
{
MethodInfo getTaskSchedulersForDebuggerMethod = typeof(TaskScheduler).GetTypeInfo().GetDeclaredMethod("GetTaskSchedulersForDebugger");
var cesp = new ConcurrentExclusiveSchedulerPair();
TaskScheduler[] foundSchedulers = getTaskSchedulersForDebuggerMethod.Invoke(null, null) as TaskScheduler[];
Assert.NotNull(foundSchedulers);
Assert.Contains(TaskScheduler.Default, foundSchedulers);
Assert.Contains(cesp.ConcurrentScheduler, foundSchedulers);
Assert.Contains(cesp.ExclusiveScheduler, foundSchedulers);
GC.KeepAlive(cesp);
}
public async Task TestChoose3_BasicFunctionality()
{
for (int chooseTestCase = 0; chooseTestCase < 5; chooseTestCase++)
{
var cesp = new ConcurrentExclusiveSchedulerPair(); // ensure BufferBlocks are serialized
var source1 = new BufferBlock<int>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
var source2 = new BufferBlock<string>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
var source3 = new BufferBlock<double>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
int intValue = 0;
string stringValue = null;
double doubleValue = 0.0;
TaskScheduler usedScheduler = null, requestedScheduler = new ConcurrentExclusiveSchedulerPair().ConcurrentScheduler;
var cts = new CancellationTokenSource();
var t = chooseTestCase < 7 ?
DataflowBlock.Choose(
source1, i => intValue = i,
source2, s => stringValue = s,
source3, d => doubleValue = d) :
DataflowBlock.Choose(
source1, i => usedScheduler = TaskScheduler.Current,
source2, s => usedScheduler = TaskScheduler.Current,
source3, d => usedScheduler = TaskScheduler.Current,
new DataflowBlockOptions { TaskScheduler = requestedScheduler, MaxMessagesPerTask = 1, CancellationToken = cts.Token });
switch (chooseTestCase)
{
case 0: // Test data on the first source
source1.PostItems(42, 43);
Assert.Equal(expected: 0, actual: await t);
Assert.Equal(expected: 42, actual: intValue);
Assert.Null(stringValue);
Assert.Equal(expected: 0, actual: doubleValue);
Assert.Equal(expected: 1, actual: source1.Count);
break;
case 1: // Test data on the second source
source2.PostItems("42", "43");
Assert.Equal(expected: 1, actual: await t);
Assert.Equal(expected: "42", actual: stringValue);
Assert.Equal(expected: 0, actual: intValue);
Assert.Equal(expected: 0, actual: doubleValue);
Assert.Equal(expected: 1, actual: source2.Count);
break;
case 2: // Test data on the third source
source3.PostItems(42.0, 43.0);
Assert.Equal(expected: 2, actual: await t);
Assert.Equal(expected: 42.0, actual: doubleValue);
Assert.Equal(expected: 0, actual: intValue);
Assert.Null(stringValue);
Assert.Equal(expected: 1, actual: source3.Count);
break;
case 3: // Test first source complete and data on second
source1.Complete();
source2.Post("42");
Assert.Equal(expected: 1, actual: await t);
Assert.Equal(expected: "42", actual: stringValue);
Assert.Equal(expected: 0, actual: intValue);
Assert.Equal(expected: 0, actual: doubleValue);
Assert.Equal(expected: 0, actual: source2.Count);
break;
case 4: // Test second source complete and data on third
source2.Complete();
source3.Post(42.0);
Assert.Equal(expected: 2, actual: await t);
Assert.Equal(expected: 42.0, actual: doubleValue);
Assert.Equal(expected: 0, actual: intValue);
Assert.Null(stringValue);
Assert.Equal(expected: 0, actual: source3.Count);
break;
case 5: // Test third source complete and data on first
source3.Complete();
source1.Post(42);
Assert.Equal(expected: 0, actual: await t);
Assert.Equal(expected: 42, actual: intValue);
Assert.Null(stringValue);
Assert.Equal(expected: 0, actual: doubleValue);
Assert.Equal(expected: 0, actual: source1.Count);
break;
case 6: // Test all sources complete
source1.Complete();
source2.Complete();
source3.Complete();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => t);
Assert.Equal(expected: 0, actual: intValue);
Assert.Null(stringValue);
Assert.Equal(expected: 0, actual: doubleValue);
break;
// >= 7 TEST USING DATAFLOW BLOCK OPTIONS
case 7: // Test correct TaskScheduler is used
source3.Post(42);
await t;
Assert.Equal(expected: requestedScheduler, actual: usedScheduler);
break;
case 8: // Test cancellation before data takes effect
cts.Cancel();
source1.Post(42);
source2.Post("43");
source3.Post(44.0);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => t);
Assert.Equal(expected: 1, actual: source1.Count);
Assert.Equal(expected: 1, actual: source2.Count);
Assert.Equal(expected: 1, actual: source3.Count);
break;
}
}
}
static void Main(string[] args)
{
started = DateTimeOffset.Now;
ServicePointManager.DefaultConnectionLimit = Int32.MaxValue;
ServicePointManager.DnsRefreshTimeout = Int32.MaxValue;
ServicePointManager.MaxServicePoints = Int32.MaxValue;
ServicePointManager.CheckCertificateRevocationList = false;
//string url = "https://dailylists.magistratesvic.com.au/EFAS/CaseSearch_GridData";
//string response;
//using (WebClient webClient = new WebClient())
//{
// webClient.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
// webClient.Headers.Add(HttpRequestHeader.UserAgent, USERAGENT);
// webClient.Headers.Add(HttpRequestHeader.Accept, "application/json; charset=utf-8");
// response = webClient.UploadString(url, "sort=&page=1&pageSize=150000&group=&filter=&CaseType=CRI&CourtID=4&HearingDate=&CourtLinkCaseNo=&PlaintiffInformantApplicant=&DefendantAccusedRespondent=&HearingType=");
//}
//ResponseData responseData = json.Deserialize<ResponseData>(response);
ConcurrentExclusiveSchedulerPair sch = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Current, Int32.MaxValue);
//Parallel.ForEach<ResponseDataHearing>(responseData.Data, new ParallelOptions { MaxDegreeOfParallelism = -1, TaskScheduler = sch.ConcurrentScheduler }, (ResponseDataHearing h) =>
//{
// string urlB = string.Format("https://dailylists.magistratesvic.com.au/EFAS/Case{0}?CaseID={1}", string.IsNullOrWhiteSpace(h.CaseType) ? "CRI" : h.CaseType, h.CaseID);
// string data;
// using (WebClient hearingClient = new WebClient())
// {
// data = hearingClient.DownloadString(urlB);
// }
// string dataRead = data.Substring(data.IndexOf("<label for=\"ProsecutingAgency\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// dataRead = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// h.ProsecutingAgency = dataRead;
// dataRead = data.Substring(data.IndexOf("<label for=\"Informant\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.Informant = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"ProsecutorRepresentative\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.ProsecutorRepresentative = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"Accused\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.Accused = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"AccusedRepresentative\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.AccusedRepresentative = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"HearingType\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.HearingType = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"Plea\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.Plea = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
// dataRead = data.Substring(data.IndexOf("<label for=\"CourtRoom\">"));
// dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
// h.CourtRoom = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
//});
//IEnumerable<CourtDate> hearings = responseData.Data
// .OrderBy(d => d.HearingDateTime)
// .GroupBy(k => k.HearingDateTime.Date)
// .Select(k => new CourtDate(k.Key, k))
// .ToArray();
Parallel.For(0, 366, new ParallelOptions {
MaxDegreeOfParallelism = -1, TaskScheduler = sch.ConcurrentScheduler
}, (int i) =>
{
DateTimeOffset procDate = started.AddDays(i);
CourtDate thisCourtDate = new CourtDate(procDate);
string urlD = GenerateDateUrl(procDate);
string responsed;
//Console.WriteLine("{0}: Started", procDate.ToString("yyyy-MMM-dd"));
using (WebClient webClient = new WebClient())
{
webClient.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
webClient.Headers.Add(HttpRequestHeader.UserAgent, USERAGENT);
//Console.WriteLine("{0}: Web Request Being Sent", procDate.ToString("yyyy-MMM-dd"));
responsed = webClient.UploadString(urlD, "sort=CourtLinkCaseNo-desc&page=1&pageSize=500&group=&filter=");
}
//Console.WriteLine("{0}: Web Response Received [{1} bytes]", procDate.ToString("yyyy-MMM-dd"), response.Length);
ResponseData responseData = json.Deserialize <ResponseData>(responsed);
//Console.WriteLine("{0}: Processing hearings [{1} total]", procDate.ToString("yyyy-MMM-dd"), responseData.Data.Length);
Parallel.ForEach <ResponseDataHearing>(responseData.Data, new ParallelOptions {
MaxDegreeOfParallelism = -1, TaskScheduler = sch.ConcurrentScheduler
},
(ResponseDataHearing h) =>
{
//Console.WriteLine("{0}-{1}: Started", procDate.ToString("yyyy-MMM-dd"), r.CourtLinkCaseNo);
string urlB = string.Format("https://dailylists.magistratesvic.com.au/EFAS/Case{0}?CaseID={1}", string.IsNullOrWhiteSpace(h.CaseType) ? "CRI" : h.CaseType, h.CaseID);
string data;
using (WebClient webClientB = new WebClient())
{
//Console.WriteLine("{0}-{1}: Sending web request", procDate.ToString("yyyy-MMM-dd"), r.CourtLinkCaseNo);
data = webClientB.DownloadString(urlB);
//Console.WriteLine("{0}-{1}: Web Response Received [{2} bytes]", procDate.ToString("yyyy-MMM-dd"), r.CourtLinkCaseNo, data.Length);
}
string dataRead = data.Substring(data.IndexOf("<label for=\"ProsecutingAgency\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
dataRead = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
h.ProsecutingAgency = dataRead;
dataRead = data.Substring(data.IndexOf("<label for=\"Informant\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.Informant = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"ProsecutorRepresentative\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.ProsecutorRepresentative = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"Accused\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.Accused = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"AccusedRepresentative\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.AccusedRepresentative = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"HearingType\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.HearingType = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"Plea\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.Plea = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
dataRead = data.Substring(data.IndexOf("<label for=\"CourtRoom\">"));
dataRead = dataRead.Substring(dataRead.IndexOf("<td>"));
h.CourtRoom = dataRead.Substring(4, dataRead.IndexOf("</td>") - 4).Trim();
//Console.WriteLine("{0}-{1}: Is a '{2}' hearing", procDate.ToString("yyyy-MMM-dd"), r.CourtLinkCaseNo, r.HearingType);
thisCourtDate.AddHearing(h);
//Console.WriteLine("{0}-{1}: done", procDate.ToString("yyyy-MMM-dd"), r.CourtLinkCaseNo);
});
courtDates.Add(thisCourtDate);
//Console.WriteLine("{0}: done", procDate.ToString("yyyy-MMM-dd"));
});
System.IO.File.WriteAllText(System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, string.Concat(started.ToString("yyyyMMdd-HHmm"), ".json")), json.Serialize(courtDates));
Console.Write("Done");
Console.Beep();
Console.ReadKey();
}
private TaskSchedulerFiber(ConcurrentExclusiveSchedulerPair concurrentExclusiveSchedulerPair) : base(concurrentExclusiveSchedulerPair.ExclusiveScheduler)
{
_ConcurrentExclusiveSchedulerPair = concurrentExclusiveSchedulerPair;
SynchronizationContext = new TaskSchedulerSynchronizationContext(concurrentExclusiveSchedulerPair.ExclusiveScheduler);
}
static async Task Main(string[] args)
{
var connectionFactory = new ConnectionFactory
{
HostName = "localhost",
Port = 5672,
UseBackgroundThreadsForIO = true
};
#region TopologyAndSending
CreateTopologyIfNecessary(InputQueue, connectionFactory);
var cts = new CancellationTokenSource();
var sendConnection = connectionFactory.CreateConnection($"{InputQueue} sender");
var senderChannel = new ConfirmsAwareChannel(sendConnection);
var sendMessagesTask = Task.Run(() => SendMessages(senderChannel, InputQueue, cts.Token), CancellationToken.None);
var receiveConnection = connectionFactory.CreateConnection($"{InputQueue} pump");
var receiveChannel = receiveConnection.CreateModel();
TaskScheduler.UnobservedTaskException += (sender, args) => { };
#endregion
receiveChannel.BasicQos(prefetchSize: 0, prefetchCount: 10, global: false);
var consumer = new EventingBasicConsumer(receiveChannel);
#region NotRelevant
consumer.Registered += Consumer_Registered;
receiveConnection.ConnectionShutdown += Connection_ConnectionShutdown;
#endregion
var exclusiveScheduler = new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler;
var maxConcurrency = 4;
var semaphore = new SemaphoreSlim(maxConcurrency);
consumer.Received += (sender,
deliverEventArgs) => Consumer_Received(deliverEventArgs,
receiveChannel,
semaphore,
exclusiveScheduler,
cts.Token);
receiveChannel.BasicConsume(InputQueue, false, ConsumerTag, consumer);
#region Stop
await Console.Error.WriteLineAsync("Press any key to stop");
Console.ReadLine();
await Console.Error.WriteLineAsync("Shutting down");
cts.Cancel();
try
{
await sendMessagesTask;
}
catch (OperationCanceledException)
{
}
while (semaphore.CurrentCount != maxConcurrency)
{
await Task.Delay(50).ConfigureAwait(false);
}
receiveChannel.Close();
receiveConnection.Close();
senderChannel.Dispose();
sendConnection.Close();
#endregion
}
/// <summary>
/// The pump for received messages.
/// </summary>
/// <param name="uri">
/// The uri.
/// </param>
/// <param name="cancellation">
/// The cancellation token source.
/// </param>
/// <returns>
/// The observable stream of messages.
/// </returns>
private static IObservableSocket CreateObservableSocket(Uri uri, CancellationTokenSource cancellation)
{
var status = new ReplaySubject <ConnectionStatus>(1);
var socket = new WebSocket4Net.WebSocket(uri.ToString());
var connected = new TaskCompletionSource <int>();
var dispatcher = new ConcurrentExclusiveSchedulerPair();
var sendScheduler = new TaskPoolScheduler(new TaskFactory(dispatcher.ExclusiveScheduler));
var receiveScheduler = new TaskPoolScheduler(new TaskFactory(dispatcher.ConcurrentScheduler));
Func <string, Task> send = message =>
{
var tcs = new TaskCompletionSource <int>();
sendScheduler.Schedule(
() =>
{
try
{
if (socket.State != WebSocket4Net.WebSocketState.Open)
{
socket.Open();
}
socket.Send(message);
tcs.SetResult(0);
}
catch (Exception e)
{
tcs.SetException(e);
}
});
;
return(tcs.Task);
};
socket.Open();
var incoming = Observable.Create <string>(
observer =>
{
status.OnNext(ConnectionStatus.Connecting);
var completed = false;
EventHandler <WebSocket4Net.MessageReceivedEventArgs> onMessage = (sender, args) => observer.OnNext(args.Message);
EventHandler <ws::SuperSocket.ClientEngine.ErrorEventArgs> onError = (sender, args) =>
{
observer.OnError(args.Exception);
completed = true;
cancellation.Cancel();
};
EventHandler onClosed = (sender, args) => cancellation.Cancel();
socket.MessageReceived += onMessage;
socket.Error += onError;
socket.Closed += onClosed;
socket.Opened += (sender, args) => connected.TrySetResult(0);
cancellation.Token.Register(
() =>
{
if (!completed)
{
observer.OnCompleted();
}
socket.MessageReceived -= onMessage;
socket.Error -= onError;
socket.Closed -= onClosed;
try
{
socket.Close();
}
catch
{
// Ignore errors closing the socket.
}
dispatcher.Complete();
});
return(Disposable.Create(cancellation.Cancel));
}).SubscribeOn(receiveScheduler).Publish().RefCount();
return(new ObservableSocket(incoming, send, status, connected.Task));
}
Run(FileInfo mainScriptFile, string[] argv,
bool inspect, bool pauseDebuggerOnStart,
IImmutableSet <string> inspectLoadSet,
Func <CancellationToken, Task <string> > f,
Process parentProcess,
bool verbose)
{
var rootDir = mainScriptFile.Directory;
Debug.Assert(rootDir != null);
var settings = new V8Settings
{
EnableDebugging =
inspect || pauseDebuggerOnStart ||
inspectLoadSet.Any(),
AwaitDebuggerAndPauseOnStart =
pauseDebuggerOnStart || inspectLoadSet.Any(),
};
using (var engine = new V8JsEngine(settings))
{
var scheduler = new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler;
var console = ConsoleService.Default;
void Load(string module)
{
var path = Path.Combine(rootDir.FullName, module);
var source = File.ReadAllText(path);
if (inspectLoadSet.Contains(source))
{
source = "debugger;" + source;
}
engine.Execute(source, module);
}
using (var host = new Host(Load, console, scheduler))
{
string FormatMessage(object sender, string message)
{
var senderName = sender is string s ? s : sender.GetType().Name;
var formatted = $"{senderName}[{Thread.CurrentThread.ManagedThreadId}]: {message}";
return(formatted.FormatFoldedLines().TrimNewLineAtTail());
}
void OnUnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs args) =>
console.Error(FormatMessage(sender, args.Exception.ToString()));
TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
var infoLog = !verbose ? null : new LogEventHandler((sender, message) =>
host.Console.Info(FormatMessage(sender, message)));
var warnLog = !verbose ? null : new ErrorLogEventHandler((sender, e, message) =>
host.Console.Warn(FormatMessage(sender, e == null ? message : string.IsNullOrEmpty(message) ? e.ToString() : message + Environment.NewLine + e)));
var errorLog = !verbose ? null : new ErrorLogEventHandler((sender, e, message) =>
host.Console.Error(FormatMessage(sender, string.IsNullOrEmpty(message) ? e.ToString() : message + Environment.NewLine + e)));
foreach (var service in new ILogSource[] { host, host.Timer, host.Xhr })
{
service.InfoLog = infoLog;
service.WarnLog = warnLog;
service.ErrorLog = errorLog;
}
var tasks = new List <NamedTask>();
void AddTask(NamedTask task)
{
lock (tasks) tasks.Add(task);
}
void RemoveTask(NamedTask task)
{
lock (tasks) tasks.Remove(task);
}
host.TaskStarting += (_, task) => AddTask(task);
host.TaskFinishing += (_, task) => RemoveTask(task);
if (verbose)
{
host.ServiceCreated += (_, service) =>
{
service.InfoLog = infoLog;
service.WarnLog = warnLog;
service.ErrorLog = errorLog;
};
}
engine.EmbedHostObject("host", host);
var initScript = GetManifestResourceStream("init.js", typeof(Program)).ReadAsText();
dynamic init = engine.Evaluate(initScript, "__init.js");
init(host, engine.Evaluate("this"), argv);
if (settings.AwaitDebuggerAndPauseOnStart)
{
console.Warn(FormatMessage(nameof(Program), "Will wait for debugger to attach."));
}
Load(mainScriptFile.Name);
void Schedule(string name, Action <AsyncTaskControl> action)
{
var task = AsyncTask.Create(name, thisTask =>
{
Exception error = null;
try
{
action(thisTask);
}
catch (Exception e)
{
error = e;
}
RemoveTask(thisTask);
switch (error)
{
case null:
thisTask.FlagSuccess();
break;
case OperationCanceledException e:
thisTask.FlagCanceled(e.CancellationToken);
break;
default:
errorLog?.Invoke(nameof(Program), error, null);
thisTask.FlagError(error);
break;
}
});
AddTask(task);
task.Start(scheduler);
}
var parentProcessTask = parentProcess.AsTask(dispose: true, p => p.ExitCode,
_ => null,
exit => exit);
while (true)
{
var readCommandTask = f(CancellationToken.None);
if (parentProcessTask != null)
{
if (parentProcessTask == await Task.WhenAny(readCommandTask, parentProcessTask))
{
break;
}
}
else
{
await readCommandTask;
}
var command = (await readCommandTask)?.Trim();
if (command == null)
{
break;
}
if (command.Length == 0)
{
continue;
}
const string ondata = "ondata";
Schedule(ondata, delegate
{
infoLog?.Invoke(nameof(Program), "STDIN: " + command);
engine.CallFunction(ondata, command);
});
}
const string onclose = "onclose";
Schedule(onclose, delegate
{
engine.Execute(@"if (typeof onclose === 'function') onclose();");
});
host.Timer.CancelAll();
host.Xhr.AbortAll();
infoLog?.Invoke(typeof(Program), "Shutting down...");
ImmutableArray <Task> tasksSnapshot;
lock (tasks)
tasksSnapshot = ImmutableArray.CreateRange(from t in tasks select t.Task);
if (await tasksSnapshot.WhenAll(TimeSpan.FromSeconds(30)))
{
Debug.Assert(tasks.Count == 0);
}
else
{
warnLog?.Invoke(typeof(Program), null, "Timed-out waiting for all tasks to end for a graceful shutdown!");
}
infoLog?.Invoke(typeof(Program), "Shutdown completed.");
TaskScheduler.UnobservedTaskException -= OnUnobservedTaskException;
}
}
}
public static Task RunExclusive(
this ConcurrentExclusiveSchedulerPair schedulerPair,
Action action)
{
return(RunActionOnScheduler(schedulerPair.ExclusiveScheduler, action));
}
void Test3()
{
var schedulerPair = new ConcurrentExclusiveSchedulerPair(
TaskScheduler.Default, maxConcurrencyLevel: 8);
TaskScheduler scheduler = schedulerPair.ConcurrentScheduler;
}
public static void TestCreationOptions(string ctorType)
{
ConcurrentExclusiveSchedulerPair schedPair = null;
//Need to define the default values since these values are passed to the verification methods
TaskScheduler scheduler = TaskScheduler.Default;
int maxConcurrentLevel = Environment.ProcessorCount;
//Based on input args, use one of the ctor overloads
switch (ctorType.ToLower())
{
case "default":
schedPair = new ConcurrentExclusiveSchedulerPair();
break;
case "scheduler":
schedPair = new ConcurrentExclusiveSchedulerPair(scheduler);
break;
case "maxconcurrent":
maxConcurrentLevel = 2;
schedPair = new ConcurrentExclusiveSchedulerPair(scheduler, maxConcurrentLevel);
break;
case "all":
maxConcurrentLevel = int.MaxValue;
schedPair = new ConcurrentExclusiveSchedulerPair(scheduler, -1 /*MaxConcurrentLevel*/, -1 /*MaxItemsPerTask*/); //-1 gets converted to Int32.MaxValue
break;
default:
throw new NotImplementedException(string.Format("The option specified {0} to create the ConcurrentExclusiveSchedulerPair is invalid", ctorType));
}
//Create the factories that use the exclusive scheduler and the concurrent scheduler. We test to ensure
//that the ConcurrentExclusiveSchedulerPair created are valid by scheduling work on them.
TaskFactory writers = new TaskFactory(schedPair.ExclusiveScheduler);
TaskFactory readers = new TaskFactory(schedPair.ConcurrentScheduler);
List <Task> taskList = new List <Task>(); //Store all tasks created, to enable wait until all of them are finished
// Schedule some dummy work that should be run with as much parallelism as possible
for (int i = 0; i < 50; i++)
{
//In the current design, when there are no more tasks to execute, the Task used by concurrentexclusive scheduler dies
//by sleeping we simulate some non trivial work that takes time and causes the concurrentexclusive scheduler Task
//to stay around for addition work.
taskList.Add(readers.StartNew(() => { var sw = new SpinWait(); while (!sw.NextSpinWillYield)
{
sw.SpinOnce();
}
}));
}
// Schedule work where each item must be run when no other items are running
for (int i = 0; i < 10; i++)
{
taskList.Add(writers.StartNew(() => { var sw = new SpinWait(); while (!sw.NextSpinWillYield)
{
sw.SpinOnce();
}
}));
}
//Wait on the tasks to finish to ensure that the ConcurrentExclusiveSchedulerPair created can schedule and execute tasks without issues
foreach (var item in taskList)
{
item.Wait();
}
//verify that maxconcurrency was respected.
if (ctorType == "maxconcurrent")
{
Assert.Equal(maxConcurrentLevel, schedPair.ConcurrentScheduler.MaximumConcurrencyLevel);
}
Assert.Equal(1, schedPair.ExclusiveScheduler.MaximumConcurrencyLevel);
//verify that the schedulers have not completed
Assert.False(schedPair.Completion.IsCompleted, "The schedulers should not have completed as a completion request was not issued.");
//complete the scheduler and make sure it shuts down successfully
schedPair.Complete();
schedPair.Completion.Wait();
//make sure no additional work may be scheduled
foreach (var schedPairScheduler in new TaskScheduler[] { schedPair.ConcurrentScheduler, schedPair.ExclusiveScheduler })
{
Exception caughtException = null;
try
{
Task.Factory.StartNew(() => { }, CancellationToken.None, TaskCreationOptions.None, schedPairScheduler);
}
catch (Exception exc)
{
caughtException = exc;
}
Assert.True(
caughtException is TaskSchedulerException && caughtException.InnerException is InvalidOperationException,
"Queueing after completion should fail");
}
}
protected override void Seed(PokemonContext context)
{
/*
* This is working and is significantly faster than previous attempts.
*
* Continuing with this pattern will lead to highly readable, yet
* very repetitive code.
*
* TODO:
* - Implement attribute writes to DBContext as a function. Perform this in Parallel
* to allow for abilityDB and moveDB to run async.
* - Fill out the body of the application. Give it some personallity.
* - Look into VS 2017 event tracking. No longer provides accurate representation.
*/
AggAbilities AbilList = new AggAbilities();
foreach (Task <Ability> a in AbilList.AllAbilities)
{
a.ContinueWith(completed => {
switch (completed.Status)
{
case TaskStatus.RanToCompletion:
PokemonContext abilityDB = new PokemonContext();
abilityDB.Abilities.Add(completed.Result);
abilityDB.SaveChanges();
break;
case TaskStatus.Faulted: break;
}
}, TaskScheduler.Default);
}
AggMoves MoveList = new AggMoves();
foreach (Task <Move> m in MoveList.AllMoves)
{
m.ContinueWith(completed => {
switch (completed.Status)
{
case TaskStatus.RanToCompletion:
PokemonContext moveDB = new PokemonContext();
moveDB.Moves.Add(completed.Result);
moveDB.SaveChanges();
break;
case TaskStatus.Faulted: break;
}
}, TaskScheduler.Default);
}
AggPokemon PokeList = new AggPokemon();
TaskScheduler scheduler = new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler;
foreach (Task <Pokedata> p in PokeList.AllPokemon)
{
p.ContinueWith(completed => {
switch (completed.Status)
{
case TaskStatus.RanToCompletion:
PokemonContext pokemonDB = new PokemonContext();
Pokedata pokedata = completed.Result;
foreach (Ability a in pokedata.Abilities)
{
pokemonDB.Abilities.Attach(a);
}
foreach (Move m in pokedata.Moves)
{
pokemonDB.Moves.Attach(m);
}
pokemonDB.Pokedatas.Add(pokedata);
pokemonDB.SaveChanges();
/*
* List<Ability> PokeAbils = pokedata.Abilities.ToList();
* pokedata.Abilities.Clear();
* List<Move> PokeMoves = pokedata.Moves.ToList();
* pokedata.Moves.Clear();
*
* pokemonDB.Pokedatas.Add(completed.Result);
* pokemonDB.SaveChanges();
*
* Pokedata currentPokemon = pokemonDB.Pokedatas.Find(pokedata.PokemonId);
*
* foreach (Ability a in PokeAbils)
* {
* currentPokemon.Abilities.Add(a);
* }
*
* foreach (Move m in PokeMoves)
* {
* currentPokemon.Moves.Add(m);
* }
*/
break;
case TaskStatus.Faulted: break;
}
}, scheduler);
}
}
public static void TestMaxItemsPerTask(int maxConcurrency, int maxItemsPerTask, bool completeBeforeTaskWait)
{
//Create a custom TaskScheduler with specified max concurrency (TrackingTaskScheduler is defined in Common\tools\CommonUtils\TPLTestSchedulers.cs)
TrackingTaskScheduler scheduler = new TrackingTaskScheduler(maxConcurrency);
//We need to use the custom scheduler to achieve the results. As a by-product, we test to ensure custom schedulers are supported
ConcurrentExclusiveSchedulerPair schedPair = new ConcurrentExclusiveSchedulerPair(scheduler, maxConcurrency, maxItemsPerTask);
TaskFactory readers = new TaskFactory(schedPair.ConcurrentScheduler); //get reader and writer schedulers
TaskFactory writers = new TaskFactory(schedPair.ExclusiveScheduler);
//These are threadlocals to ensure that no concurrency side effects occur
ThreadLocal <int> itemsExecutedCount = new ThreadLocal <int>(); //Track the items executed by CEScheduler Task
ThreadLocal <int> schedulerIDInsideTask = new ThreadLocal <int>(); //Used to store the Scheduler ID observed by a Task Executed by CEScheduler Task
//Work done by both reader and writer tasks
Action work = () =>
{
//Get the id of the parent Task (which is the task created by the scheduler). Each task run by the scheduler task should
//see the same SchedulerID value since they are run on the same thread
int id = ((TrackingTaskScheduler)scheduler).SchedulerID.Value;
if (id == schedulerIDInsideTask.Value)
{ //since ids match, this is one more Task being executed by the CEScheduler Task
itemsExecutedCount.Value = ++itemsExecutedCount.Value;
//This does not need to be thread safe since we are looking to ensure that only n number of tasks were executed and not the order
//in which they were executed. Also asserting inside the thread is fine since we just want the test to be marked as failure
Assert.True(itemsExecutedCount.Value <= maxItemsPerTask, string.Format("itemsExecutedCount={0} cant be greater than maxValue={1}. Parent TaskID={2}",
itemsExecutedCount, maxItemsPerTask, id));
}
else
{ //Since ids don't match, this is the first Task being executed in the CEScheduler Task
schedulerIDInsideTask.Value = id; //cache the scheduler ID seen by the thread, so other tasks running in same thread can see this
itemsExecutedCount.Value = 1;
}
//Give enough time for a Task to stay around, so that other tasks will be executed by the same CEScheduler Task
//or else the CESchedulerTask will die and each Task might get executed by a different CEScheduler Task. This does not affect the
//verifications, but its increases the chance of finding a bug if the maxItemPerTask is not respected
new ManualResetEvent(false).WaitOne(1);
};
List <Task> taskList = new List <Task>();
int maxConcurrentTasks = maxConcurrency * maxItemsPerTask * 5;
int maxExclusiveTasks = maxConcurrency * maxItemsPerTask * 2;
// Schedule Tasks in both concurrent and exclusive mode
for (int i = 0; i < maxConcurrentTasks; i++)
{
taskList.Add(readers.StartNew(work));
}
for (int i = 0; i < maxExclusiveTasks; i++)
{
taskList.Add(writers.StartNew(work));
}
if (completeBeforeTaskWait)
{
schedPair.Complete();
schedPair.Completion.Wait();
Assert.True(taskList.TrueForAll(t => t.IsCompleted), "All tasks should have completed for scheduler to complete");
}
//finally wait for all of the tasks, to ensure they all executed properly
Task.WaitAll(taskList.ToArray());
if (!completeBeforeTaskWait)
{
schedPair.Complete();
schedPair.Completion.Wait();
Assert.True(taskList.TrueForAll(t => t.IsCompleted), "All tasks should have completed for scheduler to complete");
}
}
public AsyncReaderWriterLock()
{
pair = new ConcurrentExclusiveSchedulerPair();
concurrentFactory = new TaskFactory(pair.ConcurrentScheduler);
exclusiveFactory = new TaskFactory(pair.ExclusiveScheduler);
}
public async Task TestSchedulerUsage()
{
foreach (bool singleProducerConstrained in DataflowTestHelpers.BooleanValues)
{
var scheduler = new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler;
var sync = new ActionBlock<int>(_ => Assert.Equal(scheduler.Id, TaskScheduler.Current.Id),
new ExecutionDataflowBlockOptions
{
TaskScheduler = scheduler,
SingleProducerConstrained = singleProducerConstrained
});
sync.PostRange(0, 10);
sync.Complete();
await sync.Completion;
var async = new ActionBlock<int>(_ => {
Assert.Equal(scheduler.Id, TaskScheduler.Current.Id);
return Task.FromResult(0);
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = scheduler,
SingleProducerConstrained = singleProducerConstrained
});
async.PostRange(0, 10);
async.Complete();
await async.Completion;
}
}
public ConcurrentExclusiveSimpleIncrementTasks()
{
m_strandExclusiveSchedulerPair = new ConcurrentStrandSchedulerPair(Environment.ProcessorCount);
m_concurrentExclusiveSchedulerPair = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount);
}
public async Task TestChoose3_Exceptions()
{
for (int test = 1; test <= 3; test++)
{
var cesp = new ConcurrentExclusiveSchedulerPair(); // ensure BufferBlocks are serialized
var source1 = new BufferBlock<int>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
var source2 = new BufferBlock<string>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
var source3 = new BufferBlock<double>(new DataflowBlockOptions { TaskScheduler = cesp.ExclusiveScheduler });
var t = DataflowBlock.Choose(
source1, i => { throw new InvalidOperationException(); },
source2, s => { throw new InvalidCastException(); },
source3, d => { throw new FormatException(); });
Assert.False(t.IsCompleted);
switch (test)
{
case 1:
source1.Post(42);
source2.Post("43");
source3.Post(44.0);
await Assert.ThrowsAsync<InvalidOperationException>(() => t);
Assert.Equal(expected: 0, actual: source1.Count);
Assert.Equal(expected: 1, actual: source2.Count);
Assert.Equal(expected: 1, actual: source3.Count);
break;
case 2:
source2.Post("43");
source1.Post(42);
source3.Post(44.0);
await Assert.ThrowsAsync<InvalidCastException>(() => t);
Assert.Equal(expected: 1, actual: source1.Count);
Assert.Equal(expected: 0, actual: source2.Count);
Assert.Equal(expected: 1, actual: source3.Count);
break;
case 3:
source3.Post(44.0);
source1.Post(42);
source2.Post("43");
await Assert.ThrowsAsync<FormatException>(() => t);
Assert.Equal(expected: 1, actual: source1.Count);
Assert.Equal(expected: 1, actual: source2.Count);
Assert.Equal(expected: 0, actual: source3.Count);
break;
}
}
}
public TaskSchedulerQueue()
{
_ConcurrentExclusiveSchedulerPair = new ConcurrentExclusiveSchedulerPair();
_Scheduler = _ConcurrentExclusiveSchedulerPair.ExclusiveScheduler;
_TaskFactory = new TaskFactory(_Scheduler);
}
