private void CheckEof()
{
if (_eofs.All(v => v.Value))
{
SendEof();
}
}
public async Task <bool> EndTuningAsync(string clientId)
{
try
{
if (!_clientChannels.TryRemove(clientId, out ChannelInfo channel))
{
return(false);
}
if (channel != null && _clientChannels.All(c => c.Value?.ChannelId != channel.ChannelId))
{
ITimeshiftControlEx timeshiftControl = ServiceRegistration.Get <ITvProvider>() as ITimeshiftControlEx;
if (!(await timeshiftControl.StopTimeshiftAsync(TV_USER_NAME, channel.SlotIndex).ConfigureAwait(false)))
{
_logger.Error("SlimTvHandler: Couldn't stop timeshifting for channel {0}", channel.ChannelId);
return(false);
}
}
return(true);
}
catch (Exception ex)
{
_logger.Error("SlimTvHandler: Error ending tuning for client {0}", ex, clientId);
}
return(false);
}
public async Task ItWorks(int messageCount, int consumers)
{
var tempDirectory = NewTempDirectory();
var client = Configure.With(Using(new BuiltinHandlerActivator()))
.Logging(l => l.Console(LogLevel.Info))
.Transport(t => t.UseFileSystemAsOneWayClient(tempDirectory))
.Routing(r => r.TypeBased().Map <string>("consumer"))
.Start();
var messages = new HashSet <string>(Enumerable.Range(0, messageCount).Select(o => $"THIS IS MESSAGE {o}"));
var messageCounts = new ConcurrentDictionary <string, int>();
consumers.Times(() => StartConsumer(messageCounts, tempDirectory));
foreach (var message in messages)
{
await client.Send(message);
}
await messageCounts.WaitUntil(d => d.Count >= messageCount, timeoutSeconds : 10);
// wait 1 extra second for unexpected messages to arrive...
await Task.Delay(TimeSpan.FromSeconds(1));
Assert.That(messages.OrderBy(m => m), Is.EqualTo(messageCounts.Keys.OrderBy(m => m)));
Assert.That(messageCounts.All(c => c.Value == 1), Is.True, $@"Not all message counts were exactly 0:
{string.Join(Environment.NewLine, messageCounts.Where(kvp => kvp.Value > 1).Select(kvp => $" {kvp.Key}: {kvp.Value}"))}
");
}
public static void RefreshAll()
{
Instances.All(i => {
i.Value.Refresh();
return(true);
});
}
/// <summary>
/// Adds a Reference.
/// </summary>
public static void Add(int Handle, int SpecificHandle, object proc)
{
#if !__IOS__
if (proc == null)
{
return;
}
if (proc.Equals(Freeproc))
{
return;
}
var key = Tuple.Create(Handle, SpecificHandle);
var contains = Procedures.ContainsKey(key);
if (Freeproc != null && Procedures.All(pair => pair.Key.Item1 != Handle))
{
Bass.ChannelSetSync(Handle, SyncFlags.Free, 0, Freeproc);
}
if (contains)
{
Procedures[key] = proc;
}
else
{
Procedures.TryAdd(key, proc);
}
#endif
}
public void Push(GyroQuaternion quat)
{
if (_calibrationDictionary.All(x => x.Key != quat.sensorName))
{
_calibrationDictionary.TryAdd(quat.sensorName, new GyroQuaternion(quat.sensorName, quat.qX, quat.qY, quat.qZ, quat.qW));
}
}
public Container ClearContextId()
{
_builders.All(x => x.Value.Clean(_contextId.Value));
_contextId.Value = new Guid();
return(this);
}
public Dictionary <string, decimal> Solve(int n = 4, decimal d = 0.85M)
{
bool converged = false;
Dictionary <string, ISet <string> > pointingDict = GetPointingDict(Graph);
ConcurrentDictionary <string, decimal> pageRankDict = new ConcurrentDictionary <string, decimal>();
var nodeArray = Graph.Keys.ToArray();
foreach (string key in Graph.Keys)
{
pageRankDict[key] = 1.0M / Graph.Count;
}
ConcurrentDictionary <string, decimal> pageRankDictLast = pageRankDict;
List <Action> actions = new List <Action>();
while (!converged)
{
pageRankDictLast = pageRankDict;
pageRankDict = new ConcurrentDictionary <string, decimal>(pageRankDictLast);
var threads = actions.Select(action => new Thread(() => action())).ToArray();
Parallel.For(0, n, i =>
{
int range = Graph.Count / 4;
int startIndex = i * range;
int endIndex = (i + 1) < n ? (i + 1) * range : Graph.Count;
for (int j = startIndex; j < endIndex; ++j)
{
string node = nodeArray[j];
decimal newPageRank = pointingDict[node].Aggregate(
0.0M,
(accumulator, node) => accumulator + pageRankDictLast[node] / Graph[node].Count
);
newPageRank = (1.0M - d) + d * newPageRank;
pageRankDict[node] = newPageRank;
}
});
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
converged = pageRankDict.All((entry) => entry.Value == pageRankDictLast[entry.Key]);
}
decimal sum = pageRankDict.Values.Sum();
Dictionary <string, decimal> result = pageRankDict.ToDictionary(entry => entry.Key, entry => entry.Value / sum);
return(result);
}
public void AddToParty(Guid guid, string username)
{
if (_knownPartyEntities.All(x => x.Key != guid))
{
_knownPartyEntities.TryAdd(guid, username);
}
SetPartyMemberUi();
}
private void TryCancel()
{
bool shouldCancel = Clients.All(pair => !pair.Value.IsConnected);
if (shouldCancel)
{
_cancellationTokenSource.Cancel();
}
}
public void DeleteAllUnknownFiles()
{
if (_fileInfos.All(fi => fi.Value.FileType != KVFileType.Unknown))
{
return;
}
foreach (var fileId in _fileInfos.Where(fi => fi.Value.FileType == KVFileType.Unknown).Select(fi => fi.Key).ToArray())
{
_fileCollection.GetFile(fileId).Remove();
_fileInfos.TryRemove(fileId);
}
}
public async Task SendAndReceiveManyMessages(int messageCount)
{
var allMessagesReceived = new ManualResetEvent(false);
var idCounts = new ConcurrentDictionary <int, int>();
var sentMessages = 0;
var receivedMessages = 0;
var stopWatch = new Stopwatch();
var inputQueueAddress = TestConfig.GetName("input1");
var bus1 = _busFactory.GetBus <MessageWithId>(inputQueueAddress, async msg =>
{
idCounts.AddOrUpdate(msg.Id, 1, (id, old) => old + 1);
Interlocked.Increment(ref receivedMessages);
if (receivedMessages >= messageCount)
{
stopWatch.Stop();
Console.WriteLine("DONE: took time:" + stopWatch.ElapsedMilliseconds + "ms");
allMessagesReceived.Set();
}
});
var messagesToSend = Enumerable.Range(0, messageCount)
.Select(id => new MessageWithId(id))
.ToList();
using (var printTimer = new Timer(5000))
{
printTimer.Elapsed += delegate { Console.WriteLine("Sent: {0}, Received: {1}", sentMessages, receivedMessages); };
printTimer.Start();
stopWatch.Start();
Console.WriteLine("Sending {0} messages", messageCount);
await Task.WhenAll(messagesToSend.Select(async msg =>
{
await bus1.SendLocal(msg);
Interlocked.Increment(ref sentMessages);
}));
var timeout = TimeSpan.FromSeconds(messageCount * 0.01 + 100);
Console.WriteLine("Waiting up to {0} seconds", timeout.TotalSeconds);
allMessagesReceived.WaitOrDie(timeout, errorMessageFactory: () => GenerateErrorText(idCounts));
}
Console.WriteLine("Waiting one more second in case messages are still dripping in...");
await Task.Delay(1000);
var errorText = GenerateErrorText(idCounts);
Assert.That(idCounts.Count, Is.EqualTo(messageCount), errorText);
Assert.That(idCounts.All(c => c.Value == 1), errorText);
}
/// <summary>
/// Stop monitoring of events.
/// </summary>
/// <returns></returns>
public async Task <StopResult> StopAsync()
{
if (_observedTimestamps == null)
{
return(new StopResult());
}
Interlocked.Exchange(ref _shuttingDown, 1);
var endTime = DateTime.UtcNow;
// to finish up messages related to a timestamp, we collect some more time
await Task.Delay(TimeSpan.FromSeconds(5));
// check one last time if all messages related for timestamp are received
CheckForMissingTimestamps();
bool allInExpectedInterval = _observedTimestamps.IsEmpty;
_logger.LogInformation("Number of incomplete timestamps while stopping: {IncompleteTimestamps}", _observedTimestamps.Count);
if (_cancellationTokenSource != null)
{
_cancellationTokenSource.Cancel();
_cancellationTokenSource = null;
}
// the stop procedure takes about a minute, so we fire and forget.
StopEventProcessorClientAsync().SafeFireAndForget(e => _logger.LogError(e, "Error while stopping event monitoring."));
bool allExpectedValueChanges = true;
if (_currentConfiguration.ExpectedValueChangesPerTimestamp > 0)
{
// TODO collect "expected" parameter as groups related to OPC UA nodes
allExpectedValueChanges = _valueChangesPerNodeId?.All(kvp =>
(kvp.Value / _totalValueChangesCount) == _currentConfiguration
.ExpectedValueChangesPerTimestamp) ??
false;
_logger.LogInformation("All expected value changes received: {AllExpectedValueChanges}",
allExpectedValueChanges);
}
return(new StopResult()
{
ValueChangesByNodeId = new ReadOnlyDictionary <string, int>(_valueChangesPerNodeId ?? new ConcurrentDictionary <string, int>()),
AllExpectedValueChanges = allExpectedValueChanges,
TotalValueChangesCount = _totalValueChangesCount,
AllInExpectedInterval = allInExpectedInterval,
StartTime = _startTime,
EndTime = endTime,
});
}
public async Task <HealthCheckResult> CheckHealthAsync(CancellationToken cancellationToken)
{
NuGetDownloadResult nuGetDownloadResult = await _nuGetDownloadClient.DownloadNuGetAsync(
NuGetDownloadSettings.Default,
_logger,
_httpClient.CreateClient("NuGet"),
cancellationToken);
if (!nuGetDownloadResult.Succeeded)
{
return(new HealthCheckResult(false));
}
var args = new List <string>
{
"Sources",
"-Format",
"Short"
};
var lines = new List <string>();
ExitCode exitCode = await ProcessRunner.ExecuteProcessAsync(nuGetDownloadResult.NuGetExePath,
args,
(message, _) =>
{
lines.Add(message);
_logger.Verbose("{Message}", message);
},
(message, category) => _logger.Verbose("{Category}{Message}", category, message),
(message, category) => _logger.Verbose("{Category}{Message}", category, message),
debugAction : (message, category) => _logger.Verbose("{Category}{Message}", category, message),
cancellationToken : cancellationToken);
if (!exitCode.IsSuccess)
{
return(new HealthCheckResult(false));
}
ConcurrentDictionary <Uri, bool?> nugetFeeds = GetFeedUrls(lines);
List <Task> tasks = nugetFeeds.Keys
.Select(nugetFeed => CheckFeedAsync(cancellationToken, nugetFeed, nugetFeeds))
.ToList();
await Task.WhenAll(tasks);
bool allSucceeded = nugetFeeds.All(pair => pair.Value == true);
return(new HealthCheckResult(allSucceeded));
}
public void ProfilingMD_Ex2()
{
using (var c = Create())
{
ConnectionMultiplexer conn = c;
var profiler = new ToyProfiler();
conn.RegisterProfiler(profiler);
var threads = new List <Thread>();
var perThreadTimings = new ConcurrentDictionary <Thread, List <IProfiledCommand> >();
for (var i = 0; i < 16; i++)
{
var db = conn.GetDatabase(i);
var thread =
new Thread(
delegate()
{
var threadTasks = new List <Task>();
conn.BeginProfiling(Thread.CurrentThread);
for (var j = 0; j < 1000; j++)
{
var task = db.StringSetAsync("" + j, "" + j);
threadTasks.Add(task);
}
Task.WaitAll(threadTasks.ToArray());
perThreadTimings[Thread.CurrentThread] = conn.FinishProfiling(Thread.CurrentThread).ToList();
}
);
profiler.Contexts[thread] = thread;
threads.Add(thread);
}
threads.ForEach(thread => thread.Start());
threads.ForEach(thread => thread.Join());
Assert.AreEqual(16, perThreadTimings.Count);
Assert.IsTrue(perThreadTimings.All(kv => kv.Value.Count == 1000));
}
}
private void OnMatch(SearchLib.Match match, int matchCount)
{
var fUpdateWordCount = new Action <Match>((m) =>
{
if (!lbTargets.Items.Cast <TargetWord>().Any(w => w.Word.ToLower() == m.Word.ToLower()))
{
var targetWord = new TargetWord(m.Word, 1);
lbTargets.Items.Add(targetWord);
}
else
{
for (int i = 0; i < lbTargets.Items.Count; i++)
{
var targetWord = lbTargets.Items[i] as TargetWord;
if (targetWord.Word.ToLower() == m.Word.ToLower())
{
targetWord.MatchCount++;
lbTargets.Items[i] = targetWord;
break;
}
}
}
});
if (this.InvokeRequired)
{
this.Invoke(fUpdateWordCount, match);
}
else
{
fUpdateWordCount(match);
}
// If we have other matches for this file, then append to that set
if (_matchResults.ContainsKey(match.File))
{
// Get matches for file
var fileMatches = _matchResults[match.File];
// Append to it
fileMatches.Matches.Add(match);
// If we're at the last file and
if (_lastFile && _matchResults.All(p => p.Value.MatchCount == p.Value.Matches.Count))
{
OnComplete();
}
}
AddMatchToResults(match, matchCount, true);
}
public async void LogoutAsync()
{
var callback = OperationContext.Current.GetCallbackChannel <IClientCallback>();
if (_clients.All(c => c.Value.Callback != callback))
{
return;
}
var taskFaktory = new TaskFactory();
await taskFaktory.StartNew(() =>
{
var clientInfo = _clients.First(c => c.Value.Callback == callback);
ClientConnection clientConnection;
if (_clients.TryRemove(clientInfo.Value.User.Login, out clientConnection))
{
foreach (var conn in _clients.Where(c => c.Value != clientConnection))
{
conn.Value.Callback.UserLoggedOut(clientConnection.User);
}
}
});
}
private Task ShardReadyAsync(ShardReadyEventArgs e)
{
_shardsReady.AddOrUpdate(e.SessionId, true, (shardKey, value) => true);
if (_shardsReady.Count == RiasBot.Shards.Count && _shardsReady.All(x => x.Value))
{
RiasBot.ShardReady -= ShardReadyAsync;
Log.Information("All shards are connected");
RiasBot.GetRequiredService <MuteService>();
var reactionsService = RiasBot.GetRequiredService <ReactionsService>();
reactionsService.WeebUserAgent = $"{RiasBot.CurrentUser.Name}/{Rias.Version}";
reactionsService.AddWeebUserAgent();
}
return(Task.CompletedTask);
}
public Task Run()
{
return(Task.Factory.StartNew(() => {
Console.WriteLine("Switch started");
while (!buffer.IsCompleted)
{
if (buffer.TryTake(out var msg, 500))
{
if (Computers.TryGetValue(msg.Address, out var computer))
{
//Console.WriteLine($"From {msg.From} : {msg.Address} - {msg.X} - {msg.Y}");
computer.Receive(msg);
}
else
{
if (NATMessage == null)
{
Console.WriteLine($"From {msg.From} : First NAT Message {msg.Address} - {msg.X} - {msg.Y}");
}
NATMessage = msg;
//Console.WriteLine($"From {msg.From} : NAT Message {msg.Address} - {msg.X} - {msg.Y}");
}
}
else
{
if (Computers.All(x => x.Value.Idle))
{
//Console.WriteLine("All computers IDLE");
if (PreviousNATMessage != null)
{
if (NATMessage.Y == PreviousNATMessage.Y)
{
Console.WriteLine($"Message twice delivered in a row : {NATMessage.Y}");
buffer.CompleteAdding();
}
}
//Console.WriteLine($"Sending NAT : {NATMessage.Address} - {NATMessage.X} - {NATMessage.Y}");
Computers[0].Receive(NATMessage);
PreviousNATMessage = NATMessage;
}
}
}
}));
}
public async Task <HealthCheckResult> CheckHealthAsync(CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(_nuGetConfiguration.NugetExePath) ||
!File.Exists(_nuGetConfiguration.NugetExePath))
{
_logger.Warning("Could not perform health checks of NuGet feeds, nuget.exe is missing");
return(new HealthCheckResult(false));
}
var args = new List <string> {
"Sources", "-Format", "Short"
};
var lines = new List <string>();
var exitCode = await ProcessRunner.ExecuteProcessAsync(_nuGetConfiguration.NugetExePath,
args,
(message, _) =>
{
lines.Add(message);
_logger.Verbose("{Message}", message);
},
(message, category) => _logger.Verbose("{Category}{Message}", category, message),
(message, category) => _logger.Verbose("{Category}{Message}", category, message),
debugAction : (message, category) => _logger.Verbose("{Category}{Message}", category, message),
cancellationToken : cancellationToken);
if (!exitCode.IsSuccess)
{
return(new HealthCheckResult(false));
}
ConcurrentDictionary <Uri, bool?> nugetFeeds = GetFeedUrls(lines);
var tasks = nugetFeeds.Keys
.Select(nugetFeed => CheckFeedAsync(nugetFeed, nugetFeeds, cancellationToken))
.ToList();
await Task.WhenAll(tasks);
bool allSucceeded = nugetFeeds.All(pair => pair.Value == true);
return(new HealthCheckResult(allSucceeded));
}
public static IEnumerable <DeviceErr> AddClient(IEnumerable <DeviceInfo> devicesList)
{
var res = new List <DeviceErr>();
foreach (var device in devicesList)
{
var deviceId = device.DeviceId;
if (_clients.ContainsKey(deviceId))
{
res.Add(new DeviceErr(deviceId, Error.DeviceIsExist));
}
}
foreach (var deviceInfo in devicesList.Where(x => _clients.All(y => y.Key != x.DeviceId)))
{
var deviceId = deviceInfo.DeviceId;
res.Add(new DeviceErr(deviceId, AddClient(deviceInfo) ? Error.Success : Error.DeviceIsExist));
}
return(res);
}
public void MoveCenters()
{
var isOptimizedMap = new ConcurrentDictionary <int, bool>();
Enumerable.Range(0, Clusters).AsParallel().ForAll(i =>
{
var clusterTotal = ClusterMap.Count(c => c.Value == i);
if (clusterTotal > 0)
{
DenseVector vectorSum = null;
for (var m = 0; m < LSA.MatrixContainer.VMatrix.ColumnCount; m++)
{
if (ClusterMap[m] == i)
{
if (vectorSum == null)
{
vectorSum = (DenseVector)LSA.MatrixContainer.VMatrix.Column(m);
}
else
{
vectorSum += (DenseVector)LSA.MatrixContainer.VMatrix.Column(m);
}
}
}
var newCenter = (vectorSum / clusterTotal).ToArray();
if (Centers[i] != null)
{
isOptimizedMap[i] = Distance.Cosine(Centers[i], newCenter) < OptimizationVarianceThreshold;
}
Centers[i] = newCenter;
}
});
IsOptimized = isOptimizedMap.All(v => v.Value == true);
}
public int Run(EventLog eventLog)
{
eventLogInstance = eventLog;
ConcurrentDictionary <string, bool> runningThreads = new ConcurrentDictionary <string, bool>();
DllManager dllManager = new DllManager();
Thread helloThread = new Thread(() => CheckForNewDll(runningThreads));
helloThread.IsBackground = true;
helloThread.Start();
Thread exportThread = new Thread(() => DoingSomething("export 11", runningThreads));
exportThread.IsBackground = true;
exportThread.Start();
Thread importThread = new Thread(() => DoingSomething("import 11", runningThreads));
importThread.IsBackground = true;
importThread.Start();
while (true)
{
Thread.Sleep(5000);
bool areAllThreadsStopped = runningThreads.All(x => x.Value == false);
if (areAllThreadsStopped)
{
LogUtil.Log("***** ALL THREADS HAVE BEEN STOPPED *****");
return(1);
}
}
}
protected virtual void Dispose(bool disposing)
{
if (disposed)
{
return;
}
if (disposing)
{
lock (this)
{
while (!clients.All(isClientFinished))
{
Monitor.Wait(this, resendTimeout);
}
}
udpclient.Close();
}
disposed = true;
}
/// <summary>
/// Gets and updates the fill forward resolution by checking specified subscription configurations and
/// selecting the smallest resoluton not equal to tick
/// </summary>
private void UpdateFillForwardResolution(FillForwardResolutionOperation operation, SubscriptionDataConfig configuration)
{
// Due to performance implications let's be jealous in updating the _fillForwardResolution
if (ValidateFillForwardResolution(configuration) &&
(
(new[] { FillForwardResolutionOperation.BeforeAdd, FillForwardResolutionOperation.AfterAdd }.Contains(operation) &&
configuration.Increment != _fillForwardResolution.Value) // check if the new Increment is different
||
(operation == FillForwardResolutionOperation.AfterRemove && // We are removing
configuration.Increment == _fillForwardResolution.Value && // True: We are removing the resolution we were using
_subscriptions.All(x => x.Key.Resolution != configuration.Resolution))) // False: there is at least another one equal, no need to update
)
{
var configurations = (operation == FillForwardResolutionOperation.BeforeAdd)
? _subscriptions.Keys.Concat(new[] { configuration }) : _subscriptions.Keys;
_fillForwardResolution.Value = configurations.Where(ValidateFillForwardResolution)
.Select(x => x.Resolution)
.Distinct()
.DefaultIfEmpty(Resolution.Minute)
.Min().ToTimeSpan();
}
}
public static string checkMagazine(string[] magazine, string[] note)
{
// Concurrency dictionary affects performance but brings utility functions that were implemented in later versions of the .NetFramework which are currently unavailable for the online compiler
var magazineDict = new ConcurrentDictionary <string, int>();
foreach (var str in magazine)
{
magazineDict.AddOrUpdate(str, 1, (x, count) => count + 1);
}
var noteDictionary = new ConcurrentDictionary <string, int>();
foreach (var str in note)
{
noteDictionary.AddOrUpdate(str, 1, (x, count) => count + 1);
}
var result = noteDictionary.All(x => magazineDict.ContainsKey(x.Key) && magazineDict[x.Key] >= x.Value);
Console.WriteLine(result ? "Yes" : "No");
return(result ? "Yes" : "No");
}
/// <summary>
/// Gets a connection for a new subscription or query. Can be an existing if there are open position or a new one.
/// </summary>
/// <param name="address">The address the socket is for</param>
/// <param name="authenticated">Whether the socket should be authenticated</param>
/// <returns></returns>
protected virtual SocketConnection GetWebsocket(string address, bool authenticated)
{
var socketResult = sockets.Where(s => s.Value.Socket.Url == address && (s.Value.Authenticated == authenticated || !authenticated) && s.Value.Connected).OrderBy(s => s.Value.HandlerCount).FirstOrDefault();
var result = socketResult.Equals(default(KeyValuePair <int, SocketConnection>)) ? null : socketResult.Value;
if (result != null)
{
if (result.HandlerCount < SocketCombineTarget || (sockets.Count >= MaxSocketConnections && sockets.All(s => s.Value.HandlerCount >= SocketCombineTarget)))
{
// Use existing socket if it has less than target connections OR it has the least connections and we can't make new
return(result);
}
}
// Create new socket
var socket = CreateSocket(address);
var socketWrapper = new SocketConnection(this, socket);
foreach (var kvp in genericHandlers)
{
socketWrapper.AddHandler(kvp.Key, false, kvp.Value);
}
return(socketWrapper);
}
public bool IsFullySynched()
{
return(_Summaries.All(s => s.Value.Status != null && s.Value.Status.IsFullySynched));
}
public SetupGameState(
Lobby lobby,
int numExpectedPerUser,
string unityTitle = "Setup Time!",
string unityInstructions = "",
TimeSpan?setupDuration = null)
: base(lobby: lobby, exit: new WaitForUsers_StateExit(lobby))
{
ConcurrentDictionary <User, int> usersToNumSubmitted = new ConcurrentDictionary <User, int>();
foreach (User user in lobby.GetAllUsers())
{
usersToNumSubmitted.AddOrReplace(user, 0);
}
StateChain setupChain = new StateChain(
stateGenerator: (int counter) =>
{
if (counter < numExpectedPerUser)
{
SimplePromptUserState setupUserState = new SimplePromptUserState(
promptGenerator: (User user) =>
{
return(CountingPromptGenerator(user, usersToNumSubmitted[user]));
},
formSubmitHandler: (User user, UserFormSubmission input) =>
{
(bool, string)handlerResponse = CountingFormSubmitHandler(user, input, usersToNumSubmitted[user]);
if (handlerResponse.Item1)
{
usersToNumSubmitted[user]++;
}
return(handlerResponse);
},
userTimeoutHandler: (User user, UserFormSubmission input) =>
{
return(CountingUserTimeoutHandler(user, input, usersToNumSubmitted[user]));
});
setupUserState.AddPerUserExitListener((User user) =>
{
if (usersToNumSubmitted.All(kvp => kvp.Value >= numExpectedPerUser)) // if after this users submission everyone has finished the expected amount it rushes everyone through
{
this.HurryUsers();
}
});
return(setupUserState);
}
else
{
return(null);
}
},
stateDuration: setupDuration);
this.Entrance.Transition(setupChain);
setupChain.Transition(this.Exit);
this.Legacy_UnityView = new Legacy_UnityView(lobby)
{
ScreenId = new StaticAccessor <TVScreenId> {
Value = TVScreenId.WaitForUserInputs
},
Title = new StaticAccessor <string> {
Value = unityTitle
},
Instructions = new StaticAccessor <string> {
Value = unityInstructions
},
};
}
public BermudaResult GetData(string domain, string query, string mapreduce, string merge, string paging, int remdepth, string command, string cursor, string paging2)
{
var args = ParseCommand(command);
bool noCache = args.Contains("-nocache");
bool makeCursor = cursor == MakeCursorToken;
bool useCursor = !makeCursor && !string.IsNullOrWhiteSpace(cursor);
DateTime minDate = DateTime.MinValue;
DateTime maxDate = DateTime.MaxValue;
if (remdepth > 0)
{
//map
var queryHash = cursor ?? GetQueryHash(domain, query, mapreduce, merge, paging, null);
//reduce
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
#if DEBUG
if (CacheTraceMessageLevel < 3) Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
#endif
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return GetCursorData(paging2, cachedDatapoints, dataType);
}
else
{
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_1" }, CacheKey = cachedDatapoints.CacheKey };
}
}
else
{
if (useCursor) throw new Exception("Cursor " + cursor + " not found");
//var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
var reducers = HostEnvironment.Instance.GetAvailablePeerConnections();
if (!reducers.Any()) throw new Exception("Specified dataset not loaded: " + domain);
ConcurrentDictionary<PeerInfo, BermudaResult> results = new ConcurrentDictionary<PeerInfo, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List<Task> tasks = new List<Task>();
foreach (var reducer in reducers)
{
Task t = new Task((peerObj) =>
{
var peerInfo = peerObj as PeerInfo;
var initiated = DateTime.Now;
var subqueryHash = GetQueryHash(domain, query, mapreduce, merge, paging, peerInfo.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(subqueryHash, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
BermudaResult res = null;
if (useCursor)
{
var dataType2 = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
res = GetCursorData(paging2, cachedDatapoints2, dataType2);
}
else
{
res = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_2" } };
}
results[peerInfo] = res;
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (peerInfo.Equals(Endpoint))
{
subresult = GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
else
{
using (var client = HostEnvironment.GetServiceClient(peerInfo))
{
subresult = client.GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
//subresult = GetDataFromPeer(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, command, assignment.PeerEndpoint.Endpoint);
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.Metadata.Initiated = initiated;
subresult.Metadata.Completed = DateTime.Now;
subresult.Metadata.OperationTime = sw2.Elapsed;
results[peerInfo] = CachedData[subqueryHash] = subresult;
}
catch (Exception ex)
{
results[peerInfo] = new BermudaResult { Error = "[Failed Node] " + ex };
}
}
}, reducer, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
#if DEBUG
Trace.WriteLine("Join Time:" + sw.Elapsed);
#endif
if (results.Any(x => x.Value.Error != null)) throw new BermudaException("Some nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
if (results.All(x => x.Value.Data == null)) return new BermudaResult { Metadata = new BermudaNodeStatistic { Notes = "No Data" } };
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1) throw new BermudaException("Subresults must all return the same type");
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null) return new BermudaResult { Error = "Could not determine the merge type, none of the nodes provided type info" };
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Where(x => !string.IsNullOrWhiteSpace(x.Data)).Select(x => x.Data.Trim('[', ']')).Where(x => !string.IsNullOrWhiteSpace(x))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
var pagingFunc = GetPagingFunc(paging, dataType);
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
allItems = pagingInvokeMethod.Invoke(pagingFunc, new object[] { allItems });
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic { Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = HostEnvironment.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.Metadata).ToArray() };
var arraylol = ToArrayCollection(allItems, dataType);
var json = JsonConvert.SerializeObject(arraylol);
//var json = JsonConvert.SerializeObject(allItems);
var originalData = makeCursor ? arraylol : null;
var finalResult = new BermudaResult { DataType = dataTypeDescriptor, OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = finalMetadata, CacheKey = queryHash };
CachedData[queryHash] = finalResult;
return finalResult;
}
}
else
{
ConcurrentDictionary<string, BermudaResult> results = new ConcurrentDictionary<string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var bucketInterfaces = HostEnvironment.Instance.GetBucketInterfacesForDomain(domain);
if (!bucketInterfaces.Any()) throw new BermudaException("Data not loaded for: " + domain);
if (bucketInterfaces.Count() > 1) throw new BermudaException("Multiple buckets not supported by BermudaMapReduce");
var queryHash = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return GetCursorData(paging2, cachedDatapoints, dataType);
}
else
{
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_3" }, CacheKey = queryHash };
}
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
string json = null;
foreach (var bucketInterface in bucketInterfaces)
{
var bucketKey = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(bucketKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1) Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
if (useCursor)
{
if (cachedDatapoints2.OriginalData == null) throw new Exception("Cursor " + cursor + " contains null data");
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
results[bucketInterface.Name] = GetCursorData(paging2, cachedDatapoints2, dataType);
}
else
{
results[bucketInterface.Name] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_4" } };
json = cachedDatapoints2.Data;
}
}
else
{
//get mentions
var collections = GetCollections(query, mapreduce);
if (collections.Count() > 1) throw new BermudaException("More than one collection specified: " + string.Join(",", collections));
var table = collections.FirstOrDefault();
var tableName = table == null ? null : table.Source;
var raw = bucketInterface.GetData(tableName);
//var rawType = raw.GetType();
//itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
itemType = bucketInterface.GetDataType(tableName);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var pagingFunc = GetPagingFunc(paging, resultType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw;
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (json == null)
{
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
subresult = pagingInvokeMethod.Invoke(pagingFunc, new object[] { subresult });
}
datapoints = subresult;
}
//format a metada string
if (!args.Contains("-nocount"))
{
stats.TotalItems = bucketInterface.GetCount(tableName);
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = HostEnvironment.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null) return new BermudaResult() { Metadata = new BermudaNodeStatistic { Notes = "No Results" } };
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = GetMergeFunc(merge, mapreduce, itemType, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
stats.LinqExecutionTime = sw.Elapsed;
var arraylol = ToArrayCollection(datapoints, resultType);
if (json == null && datapoints != null)
{
json = JsonConvert.SerializeObject(arraylol);
}
//var json = JsonConvert.SerializeObject(datapoints);
var originalData = makeCursor ? arraylol : null;
var result = CachedData[queryHash] = new BermudaResult { DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = stats };
sw.Stop();
return result;
}
}
}
/// <summary>
/// Downloads blocks from <paramref name="peers"/> in parallel,
/// using the given <paramref name="blockFetcher"/> function.
/// </summary>
/// <param name="peers">A list of peers to download blocks.</param>
/// <param name="blockFetcher">A function to take demands and a peer, and then
/// download corresponding blocks.</param>
/// <param name="singleSessionTimeout">A maximum time to wait each single call of
/// <paramref name="blockFetcher"/>. If a call is timed out unsatisfied demands
/// are automatically retried to fetch from other peers.</param>
/// <param name="cancellationToken">A cancellation token to observe while waiting
/// for the task to complete.</param>
/// <returns>An async enumerable that yields pairs of a fetched block and its source
/// peer. It terminates when all demands are satisfied.</returns>
public async IAsyncEnumerable <Tuple <Block <TAction>, TPeer> > Complete(
IReadOnlyList <TPeer> peers,
BlockFetcher blockFetcher,
TimeSpan singleSessionTimeout,
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (!peers.Any())
{
throw new ArgumentException("The list of peers must not be empty.", nameof(peers));
}
var pool = new PeerPool(peers);
var queue = new AsyncProducerConsumerQueue <Tuple <Block <TAction>, TPeer> >();
var completion =
new ConcurrentDictionary <HashDigest <SHA256>, bool>(_satisfiedBlocks);
await foreach (var hashes in EnumerateChunks(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
IList <HashDigest <SHA256> > hashDigests =
hashes is IList <HashDigest <SHA256> > l ? l : hashes.ToList();
foreach (HashDigest <SHA256> hash in hashDigests)
{
completion.TryAdd(hash, false);
}
cancellationToken.ThrowIfCancellationRequested();
await pool.SpawnAsync(
async (peer, ct) =>
{
ct.ThrowIfCancellationRequested();
var demands = new HashSet <HashDigest <SHA256> >(hashDigests);
try
{
_logger.Debug(
"Request blocks {BlockHashes} to {Peer}...",
hashDigests,
peer
);
var timeout = new CancellationTokenSource(singleSessionTimeout);
CancellationToken timeoutToken = timeout.Token;
timeoutToken.Register(() =>
_logger.Debug("Timed out to wait a response from {Peer}.", peer)
);
ct.Register(() => timeout.Cancel());
try
{
ConfiguredCancelableAsyncEnumerable <Block <TAction> > blocks =
blockFetcher(peer, hashDigests, timeoutToken)
.WithCancellation(timeoutToken);
await foreach (Block <TAction> block in blocks)
{
_logger.Debug(
"Downloaded a block #{BlockIndex} {BlockHash} " +
"from {Peer}.",
block.Index,
block.Hash,
peer
);
if (Satisfy(block))
{
await queue.EnqueueAsync(
Tuple.Create(block, peer),
cancellationToken
);
}
demands.Remove(block.Hash);
}
}
catch (OperationCanceledException e)
{
if (ct.IsCancellationRequested)
{
_logger.Error(
e,
"A blockFetcher job (peer: {Peer}) is cancelled.",
peer
);
throw;
}
_logger.Debug(
e,
"Timed out to wait a response from {Peer}.",
peer
);
}
}
finally
{
if (demands.Any())
{
_logger.Verbose(
"Fetched blocks from {Peer}, but there are still " +
"unsatisfied demands ({UnsatisfiedDemandsNumber}) so " +
"enqueue them again: {UnsatisfiedDemands}.",
peer,
demands.Count,
demands
);
Demand(demands, retry: true);
}
else
{
_logger.Verbose("Fetched blocks from {Peer}.", peer);
}
}
},
cancellationToken : cancellationToken
);
}
while (!completion.All(kv => kv.Value))
{
Tuple <Block <TAction>, TPeer> pair;
try
{
pair = await queue.DequeueAsync(cancellationToken);
}
catch (InvalidOperationException)
{
break;
}
yield return(pair);
_logger.Verbose(
"Completed a block {BlockIndex} {BlockHash} from {Peer}.",
pair.Item1.Index,
pair.Item1.Hash,
pair.Item2
);
completion[pair.Item1.Hash] = true;
}
_logger.Verbose("Completed all blocks ({Number}).", completion.Count);
}
public BermudaResult GetData(string domain, IEnumerable<string> blobs, string query, string mapreduce, string merge, DateTime minDate, DateTime maxDate, int remdepth, object[] parameters, string command)
{
var args = ParseCommand(command);
if (remdepth > 0)
{
//map
var blobInterfaces = blobs == null ? AzureInterface.Instance.ListBlobs(domain, minDate.Ticks, maxDate.Ticks) : AzureInterface.Instance.GetBlobInterfacesByNames(domain, blobs);
var blobSetKey = GetQueryChecksum(domain, string.Join(",", blobInterfaces.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, null);
//reduce
BermudaResult cachedDatapoints;
if (CachedData.TryGetValue(blobSetKey, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 3) Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_1" } };
}
else
{
var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
if (!assignments.Any()) throw new Exception("Specified dataset not loaded: " + domain);
ConcurrentDictionary<IPEndPoint, BermudaResult> results = new ConcurrentDictionary<IPEndPoint, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List<Task> tasks = new List<Task>();
foreach (var ass in assignments)
{
Task t = new Task((assObj) =>
{
ZipMetadata assignment = assObj as ZipMetadata;
var initiated = DateTime.Now;
var blobSubsetKey = GetQueryChecksum(domain, string.Join(",", assignment.Blobs.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, assignment.PeerEndpoint.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (CachedData.TryGetValue(blobSubsetKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
results[assignment.PeerEndpoint] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_2" } };
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (assignment.PeerEndpoint.Equals(Endpoint))
{
subresult = GetData(domain, assignment.Blobs.Select(x => x.Name), query, mapreduce, merge, minDate, maxDate, remdepth - 1, parameters, command);
}
else
{
using (var client = AzureInterface.Instance.GetServiceClient(assignment.PeerEndpoint))
{
subresult = client.GetData(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, parameters, command);
}
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.MetadataObject.Initiated = initiated;
subresult.MetadataObject.Completed = DateTime.Now;
subresult.MetadataObject.OperationTime = sw2.Elapsed;
results[assignment.PeerEndpoint] = CachedData[blobSubsetKey] = subresult;
}
catch (Exception ex)
{
results[assignment.PeerEndpoint] = new BermudaResult { Error = "[Failed Node] " + ex };
}
}
}, ass, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
Trace.WriteLine("Join Time:" + sw.Elapsed);
if (results.All(x => x.Value.Error != null)) throw new Exception("All nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1) throw new Exception("Subresults must all return the same type");
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null) return new BermudaResult { Error = "Could not determine the merge type, none of the nodes provided type info" };
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Select(x => x.Data.Trim('[', ']'))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic { Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = AzureInterface.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.MetadataObject ).ToArray() };
var finalResult = new BermudaResult { DataType = dataTypeDescriptor, DataObject = allItems, CreatedOn = DateTime.Now.Ticks, MetadataObject = finalMetadata };
CachedData[blobSetKey] = finalResult;
return finalResult;
}
}
else
{
ConcurrentDictionary<string, BermudaResult> results = new ConcurrentDictionary<string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var blobInterfaces = AzureInterface.Instance.GetBlobInterfacesByNames(domain, blobs);
var blobSetKey = GetQueryChecksum(domain, string.Join(",", blobInterfaces.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (CachedData.TryGetValue(blobSetKey, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_3" } };
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
foreach (var blobInterface in blobInterfaces)
{
var blobKey = GetQueryChecksum(domain, blobInterface.Name, query, mapreduce, minDate, maxDate, parameters, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (CachedData.TryGetValue(blobKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1) Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
results[blobInterface.Name] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_4" } };
datapoints = cachedDatapoints2.DataObject;
}
else
{
//get mentions
var raw = blobInterface.GetData();
var rawType = raw.GetType();
itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw.AsParallel();
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
datapoints = subresult;
//format a metada string
if (!args.Contains("-nocount"))
{
//stats.TotalItems = raw.Count();
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = AzureInterface.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null) return new BermudaResult() { MetadataObject = new BermudaNodeStatistic { Notes = "No Results" } };
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = resultType == null ? null : GetMergeFunc(merge, mapreduce, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
sw.Stop();
stats.LinqExecutionTime = sw.Elapsed;
var result = CachedData[blobSetKey] = new BermudaResult { DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), DataObject = datapoints, CreatedOn = DateTime.Now.Ticks, MetadataObject = stats };
return result;
}
}
}
public void WeGetAllMessagesEvenThoughRabbitMqRestarts()
{
var messages = new ConcurrentDictionary<string, bool>();
_receiver.Handle<string>(async message =>
{
Console.WriteLine($"Received '{message}'");
await Task.Delay(500);
messages[message] = true;
});
Console.WriteLine("Sending messages...");
Enumerable.Range(0, 40)
.Select(i => $"message number {i}")
.ToList()
.ForEach(message =>
{
messages[message] = false;
_sender.Send(message).Wait();
});
Console.WriteLine("Waiting for all messages to have been handled...");
// restart RabbitMQ while we are receiving messages
ThreadPool.QueueUserWorkItem(_ =>
{
try
{
Thread.Sleep(5000);
Console.WriteLine("Stopping RabbitMQ....");
Exec("net", "stop rabbitmq");
Thread.Sleep(1000);
Console.WriteLine("Starting RabbitMQ....");
Exec("net", "start rabbitmq");
}
catch (Exception exception)
{
throw new AssertionException("Exception on background thread", exception);
}
});
var stopwatch = Stopwatch.StartNew();
while (true)
{
Thread.Sleep(100);
if (messages.All(kvp => kvp.Value))
{
Console.WriteLine("All messages received :)");
break;
}
if (stopwatch.Elapsed < TimeSpan.FromSeconds(40)) continue;
throw new TimeoutException("Waited too long!");
}
}
public void ProfilingMD_Ex2()
{
using (var c = Create())
{
ConnectionMultiplexer conn = c;
var profiler = new ToyProfiler();
conn.RegisterProfiler(profiler);
var threads = new List<Thread>();
var perThreadTimings = new ConcurrentDictionary<Thread, List<IProfiledCommand>>();
for (var i = 0; i < 16; i++)
{
var db = conn.GetDatabase(i);
var thread =
new Thread(
delegate()
{
var threadTasks = new List<Task>();
conn.BeginProfiling(Thread.CurrentThread);
for (var j = 0; j < 1000; j++)
{
var task = db.StringSetAsync("" + j, "" + j);
threadTasks.Add(task);
}
Task.WaitAll(threadTasks.ToArray());
perThreadTimings[Thread.CurrentThread] = conn.FinishProfiling(Thread.CurrentThread).ToList();
}
);
profiler.Contexts[thread] = thread;
threads.Add(thread);
}
threads.ForEach(thread => thread.Start());
threads.ForEach(thread => thread.Join());
Assert.AreEqual(16, perThreadTimings.Count);
Assert.IsTrue(perThreadTimings.All(kv => kv.Value.Count == 1000));
}
}
