async Task LoadRequests(CancellationToken token)
{
var partitionKeyFilter = TableQuery.GenerateFilterCondition(nameof(RequestRecord.PartitionKey), QueryComparisons.Equal, Constants.PartitionKey);
var statusFilter = TableQuery.GenerateFilterCondition(nameof(RequestRecord.Status), QueryComparisons.Equal, Status.Pending.ToString());
var filter = TableQuery.CombineFilters(partitionKeyFilter, TableOperators.And, statusFilter);
var query = new TableQuery <RequestRecord>()
.Where(filter);
var records = new List <RequestRecord>();
TableContinuationToken continuationToken = null;
do
{
var tableQuerySegment = await table.ExecuteQuerySegmentedAsync(query, continuationToken, token)
.ConfigureAwait(false);
continuationToken = tableQuerySegment.ContinuationToken;
records.AddRange(tableQuerySegment);
} while (continuationToken != null);
foreach (var record in records)
{
if (toProcess.All(r => r.RequestId != record.RequestId))
{
toProcess.Enqueue(record);
}
}
}
private void AddPeer(KnownPeer newPeer)
{
if (_peerRoundRobin.All(x => x.ConnectionString != newPeer.ConnectionString))
{
_peerRoundRobin.Enqueue(newPeer);
}
}
public void TestThatGenerationIsThreadSafeAndUnique()
{
var results = new ConcurrentQueue <string>();
// Abuse threading here to ensure safety
for (var tryNumber = 0; tryNumber < TryCount; tryNumber++)
{
var generator = new TimeBasedIdGenerator();
Parallel.For(0, EmailCount, currentMail =>
{
var email = _emails[currentMail];
var generated = generator.GenerateId(email);
results.Enqueue(generated);
}
);
// Tolerate the risk of having the same email within the same 200 ms window of a given day
Thread.Sleep(200);
}
Assert.AreEqual(EmailCount * TryCount, results.Count);
var uniqueIds = new HashSet <string>(results);
Assert.IsTrue(uniqueIds.Count <= results.Count);
// Accept a 5 delta
Assert.IsTrue(uniqueIds.Count >= results.Count - 5);
Assert.IsTrue(results.All(x => x.Length == 9));
}
public async Task CreateLogIfNotExistAppendAsync(string path, string content)
{
_requests.Enqueue(content);
if (_loggingTasks.All(t => t.IsCompleted) || !_loggingTasks.Any())
{
StringBuilder logBuilder = new StringBuilder();
while (_requests.Any())
{
if (_requests.TryDequeue(out string log))
{
logBuilder.AppendLine(log);
}
}
if (!string.IsNullOrWhiteSpace(logBuilder.ToString()))
{
_loggingTasks.Enqueue(CreateIfNotExistAppendAsync(path, logBuilder.ToString()));
}
}
else
{
await Task.WhenAll(_loggingTasks);
_loggingTasks.TryDequeue(out Task doneTask);
}
}
public void ScheduleUpdate(List <ServiceUnit> units)
{
foreach (var unit in units)
{
if (unitsUpdating.All(u => u.ServiceName != unit.ServiceName))
{
unitsUpdating.Enqueue(unit);
}
}
}
public void Test2()
{
Thread firtsThread = new Thread(fun1);
Thread secondThread = new Thread(fun1);
Thread thirdThread = new Thread(fun1);
Thread thread4 = new Thread(fun1);
Thread thread5 = new Thread(fun1);
Thread thread6 = new Thread(fun1);
Thread thread7 = new Thread(fun1);
Thread thread8 = new Thread(fun1);
Thread thread9 = new Thread(fun1);
Thread thread10 = new Thread(fun1);
firtsThread.Start();
secondThread.Start();
thirdThread.Start();
thread4.Start();
thread5.Start();
thread6.Start();
thread7.Start();
thread8.Start();
thread9.Start();
thread10.Start();
firtsThread.Join();
secondThread.Join();
secondThread.Join();
thread4.Join();
thread5.Join();
thread6.Join();
thread7.Join();
thread8.Join();
thread9.Join();
thread10.Join();
Assert.AreEqual(10, _results.Count);
Assert.AreEqual(true, _results3.All(e => e != null));
}
public void StringSplitterTest()
{
StringSplitter stringSplitter = new StringSplitter("\"");
stringSplitter.Delimiter = ",";
ConcurrentQueue <string> input = new ConcurrentQueue <string>();
ConcurrentQueue <object[]> output = new ConcurrentQueue <object[]>();
ManualResetEvent pauseButton = new ManualResetEvent(true);
Progress <int> progress = new Progress <int>();
int numinput = 2000;
for (int i = 0; i < numinput; i++)
{
input.Enqueue("\"Input, number\"," + i);
}
totalrecords = 0;
progress.ProgressChanged += progresshandler;
var action = stringSplitter.GetReportingPausableWorkItem();
List <Task> WorkList = new List <Task>();
for (int i = 0; i < 2; i++)
{
WorkList.Add(Task.Factory.StartNew(() => action(input, output, pauseButton, progress)));
}
while (!input.IsEmpty)
{
Task.Delay(200).Wait();
}
stringSplitter.SignalCompletion();
Task.WaitAll(WorkList.ToArray());
Assert.IsTrue(input.IsEmpty);
Assert.IsTrue(output.Count == numinput);
Assert.IsTrue(output.All(x => x.ElementAt(0).Equals("Input, number")));
var linqtime = output.AsEnumerable();
IEnumerable <int> numbers = linqtime.Select(x => Int32.Parse((string)x.ElementAt(1))); //forgive the weird casting it used to be a string collection
int Sum = numbers.Sum();
IEnumerable <int> checknumbers = Enumerable.Range(0, numinput); //range is non inclusive
int CheckSum = checknumbers.Sum();
Assert.IsTrue(Sum == CheckSum);
}
static void AddInnerLinks(string html, ConcurrentQueue <string> queue)
{
if (queue.Count < MAX_QUEUE_LENGTH && html != null)
{
var document = new HtmlParser().Parse(html);
var innerURLList = document.QuerySelectorAll("a").Select(element => element.GetAttribute("href")).ToList();
foreach (string url in innerURLList)
{
bool isValidLink = Uri.IsWellFormedUriString(url, UriKind.Absolute);
if (isValidLink && queue.All(item => item != url))
{
queue.Enqueue(url);
}
;
}
}
}
public void Dispose_should_stop_scheduler()
{
var intervalInMs = 50;
var watch = new Stopwatch();
var calls = new ConcurrentQueue<TimeSpan>();
_processor.Setup(p => p.ProcessSlaTimeouts()).Callback(() => { calls.Enqueue(watch.Elapsed); });
watch.Start();
using (new SlaTimeoutValidationScheduler(_processor.Object, TimeSpan.FromMilliseconds(intervalInMs)))
Thread.Sleep(intervalInMs * 2);
var timeAfterDispose = watch.Elapsed;
Thread.Sleep(intervalInMs * 2);
Assert.That(calls.Count, Is.AtLeast(1));
Assert.That(calls.All(callTime => callTime < timeAfterDispose), "Some calls has been made after Dispose()");
}
public void SplitAfter_should_not_create_a_subflow_when_no_element_is_left()
{
var result = new ConcurrentQueue <ImmutableList <Tuple <bool, int> > >();
Source.From(new[]
{
Tuple.Create(true, 1), Tuple.Create(true, 2), Tuple.Create(false, 0),
Tuple.Create(true, 3), Tuple.Create(true, 4), Tuple.Create(false, 0),
Tuple.Create(true, 5), Tuple.Create(false, 0)
})
.SplitAfter(t => !t.Item1)
.Where(t => t.Item1)
.Aggregate(ImmutableList.Create <Tuple <bool, int> >(), (list, b) => list.Add(b))
.To(Sink.ForEach <ImmutableList <Tuple <bool, int> > >(list => result.Enqueue(list)))
.Run(Materializer);
Thread.Sleep(500);
result.All(l => l.Count > 0).Should().BeTrue();
}
/// <summary>
/// This is excuted in separate thread
/// </summary>
/// <returns></returns>
private async Task DoMonitorAsync()
{
// Still there are too many emails in the queue => Ignore
if (_emailContentQueue.Count < 1000)
{
_timer.Stop();
var emails = await _mailRepository.GetNotCheckedEmailsAsync();
emails.ToList().ForEach(s =>
{
if (_emailContentQueue.All(t => t.EmailContentID != s.EmailContentID))
{
_emailContentQueue.Enqueue(s);
EnqueueProgress.Report(s);
}
});
_timer.Start();
}
}
public static void SendMessage(SEBXULMessage message)
{
try
{
if (XULRunner != null)
{
Console.WriteLine(JsonConvert.SerializeObject(message));
Logger.AddInformation("WebSocket: Send message: " + JsonConvert.SerializeObject(message));
XULRunner.Send(JsonConvert.SerializeObject(message));
}
else if (messageQueue.All(m => m != message))
{
Logger.AddInformation("WebSocket: Added message to queue: " + JsonConvert.SerializeObject(message));
messageQueue.Enqueue(message);
}
}
catch (Exception)
{
}
}
public void Dispose_should_stop_scheduler()
{
var intervalInMs = 50;
var watch = new Stopwatch();
var calls = new ConcurrentQueue <TimeSpan>();
_processor.Setup(p => p.ProcessSlaTimeouts()).Callback(() => { calls.Enqueue(watch.Elapsed); });
watch.Start();
using (new SlaTimeoutValidationScheduler(_processor.Object, TimeSpan.FromMilliseconds(intervalInMs)))
Thread.Sleep(intervalInMs * 2);
var timeAfterDispose = watch.Elapsed;
Thread.Sleep(intervalInMs * 2);
Assert.That(calls.Count, Is.AtLeast(1));
Assert.That(calls.All(callTime => callTime < timeAfterDispose), "Some calls has been made after Dispose()");
}
public bool Ping()
{
var answers = new ConcurrentQueue <bool>();
try
{
Parallel.ForEach(CacheClients, client =>
{
var oneResult = client.Ping();
answers.Enqueue(oneResult);
});
}
catch (AggregateException e)
{
if (e.InnerException != null)
{
throw e.InnerException;
}
}
return(answers.All(a => a));
}
private void DiscardExpiredItems()
{
var now = Clock.Now;
while (_queue.Any())
{
DateTimeOffset next;
if (_queue.TryPeek(out next))
{
if (next + _interval < now)
{
DateTimeOffset discard;
_queue.TryDequeue(out discard);
}
else
{
break;
}
}
}
Contract.Ensures(_queue.All(i => i + _interval >= now));
}
public async Task CanCustomizeHowManyEventsGetDispatchedEachTime(int totalCount, int minimumBatchSize, int maximumBatchSize)
{
Assert.That(totalCount % minimumBatchSize, Is.EqualTo(0),
"Please ensure that the total count is a multiple of the minimum batch size");
var topic = _factory.GetNewTopic();
var producer = _factory.ConfigureProducer().Create();
Using(producer);
var encounteredBatchSizes = new ConcurrentQueue <int>();
var consumer = _factory.ConfigureConsumer("default")
.Topics(t => t.Subscribe(topic))
.Positions(p => p.StoreInMemory())
.Options(o =>
{
o.SetMinimumBatchSize(minimumBatchSize);
o.SetMaximumBatchSize(maximumBatchSize);
o.SetMaximumPrefetchQueueLength(maximumBatchSize * 2);
})
.Handle(async(batch, context, cancellationToken) => encounteredBatchSizes.Enqueue(batch.Count))
.Create();
Using(consumer);
var messages = Enumerable.Range(0, totalCount).Select(n => $"THIS IS MESSAGE NUMBNER {n}");
using (new TimerScope($"send {totalCount} messages", totalCount))
{
await Task.WhenAll(messages.Select(m => producer.Send(topic, new ToposMessage(m))));
}
consumer.Start();
string FormatBatchSizes() => string.Join(Environment.NewLine, encounteredBatchSizes
.Select(e => $" {e}" + ((e > maximumBatchSize || e < minimumBatchSize) ? " !!!!!!!!!" : "")));
using (new TimerScope($"receive {totalCount} messages", totalCount))
using (new PeriodicCallback(TimeSpan.FromSeconds(5), () => Console.WriteLine($"{DateTime.Now:HH:mm:ss} SUM(encountered batch sizes) = {encounteredBatchSizes.Sum()}")))
{
await encounteredBatchSizes
.WaitOrDie(
completionExpression : q => q.Sum() == totalCount,
failExpression : q => q.Sum() > totalCount,
failureDetailsFunction : () => $@"The sum is {encounteredBatchSizes.Sum()}
All the batch sizes are here:
{FormatBatchSizes()}
SUM: {encounteredBatchSizes.Sum()}
",
timeoutSeconds : 20
);
}
Assert.That(encounteredBatchSizes.All(c => c <= maximumBatchSize && c >= minimumBatchSize), Is.True,
$@"Expected all encountered batch sizes N to satisfy {minimumBatchSize} <= N <= {maximumBatchSize}, but we got these:
{FormatBatchSizes()}
SUM: {encounteredBatchSizes.Sum()}
");
}
public Boolean Ready()
{
return(_agents.All(x => x.Progress == 100));
}
