private async Task LoadExistingPositions() { if (_activePositionWorkers?.Any() == true) { throw new BusinessException("Positions already loaded"); } _activePositionWorkers = new ConcurrentDictionary <string, TradingPositionWorker>(); var storedOrderPairs = _orderRepository.GetAll().ToList().GroupOrders(); foreach (var storedOrderTuple in storedOrderPairs) { var currencyPair = await _stockRestConnector.GetCurrencyPair(storedOrderTuple.Item1.CurrencyPair); if (currencyPair == null) { throw new BusinessException("Currency pair not found") { Details = $"Currency pair id: {storedOrderTuple.Item1.CurrencyPair}" } } ; var worker = await _tradingPositionWorkerFactory.CreateWorkerWithExistingPosition(storedOrderTuple.ToTradingPosition(currencyPair), OnPositionChanged); _activePositionWorkers.TryAdd(currencyPair.Id, worker); } }
void LoadSymbols() { MarketSymbols = new ConcurrentDictionary<string, string>(); resourceName = new [] { "Windows.Controls.Resources.NYSE.csv", "Windows.Controls.Resources.NASDAQ.csv", "Windows.Controls.Resources.AMEX.csv" }; var asm = Assembly.GetExecutingAssembly(); Parallel.ForEach(resourceName, (s) => ReadResourceSymbolFile(s, asm)); if (!MarketSymbols.Any()) return; Symbols = MarketSymbols.OrderBy(x => x.Key).ToDictionary(k => k.Key, v => v.Value); for (int i = 0; i < Symbols.Count; i++) { var itm = Symbols.ElementAt(i); var idx = int.Parse(itm.Value.Substring(0, 1)); // Stripping the market index char var val = itm.Value.Substring(1); Symbols[itm.Key] = val; this.AppendValues(itm.Key, val, icon[idx]); } }
public static void AddListener(SwatServer server, DiscordChannel channel) { if (_listeners.Count > 10 || _listeners.Any(kvp => kvp.Value.Count > 10)) { throw new Exception("Maximum amount of simultanous checks reached. Please try again later."); } lock (_lock) { _listeners.AddOrUpdate( server, new ConcurrentHashSet <DiscordChannel>() { channel }, (k, v) => { v.Add(channel); return(v); } ); } Start(); }
private async Task <ConcurrentDictionary <Project, ConcurrentQueue <ValueTuple <ISymbol, IReferenceFinder> > > > CreateProjectMapAsync( ConcurrentSet <ISymbol> symbols) { using (Logger.LogBlock(FunctionId.FindReference_CreateProjectMapAsync, this.cancellationToken)) { Func <Project, ConcurrentQueue <ValueTuple <ISymbol, IReferenceFinder> > > createQueue = p => new ConcurrentQueue <ValueTuple <ISymbol, IReferenceFinder> >(); var projectMap = new ConcurrentDictionary <Project, ConcurrentQueue <ValueTuple <ISymbol, IReferenceFinder> > >(); #if PARALLEL Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () => { symbols.AsParallel().WithCancellation(cancellationToken).ForAll(s => { finders.AsParallel().WithCancellation(cancellationToken).ForAll(f => { var projects = f.DetermineProjectsToSearch(s, solution, cancellationToken) ?? SpecializedCollections.EmptyEnumerable <Project>(); foreach (var project in projects.Distinct()) { projectMap.GetOrAdd(project, createQueue).Enqueue(ValueTuple.Create(s, f)); } }); }); }); #else var scope = this.documents != null?this.documents.Select(d => d.Project).ToImmutableHashSet() : null; foreach (var s in symbols) { foreach (var f in finders) { this.cancellationToken.ThrowIfCancellationRequested(); var projects = await f.DetermineProjectsToSearchAsync(s, solution, scope, cancellationToken).ConfigureAwait(false) ?? SpecializedCollections.EmptyEnumerable <Project>(); foreach (var project in projects.Distinct().WhereNotNull()) { if (scope == null || scope.Contains(project)) { projectMap.GetOrAdd(project, createQueue).Enqueue(ValueTuple.Create(s, f)); } } } } #endif Contract.ThrowIfTrue(projectMap.Any(kvp => kvp.Value.Count != kvp.Value.ToSet().Count)); return(projectMap); } }
private void OnOpened() { Debug.WriteLine("{0} WebSocket opened.", DateTime.Now); _wsReconnectionTimer.Change(Timeout.Infinite, Timeout.Infinite); // stop if (_webSocketSources.Any()) { Debug.WriteLine("{0} WebSocket recover subscriptions.", DateTime.Now); _webSocketSources.Values.ForEach(source => { source.Subscribe(); }); // resubscribe } _openedEvent.Set(); }
public ChatSession Dequeue() { if (queue.Any()) { var nextChat = queue.FirstOrDefault(x => x.Value.ChatStatus == Contracts.Data.Enums.ChatStatus.Queued).Value; if (nextChat != null) { nextChat.MarkAsProcessing(); return(nextChat); } } return(null); }
/// <summary> /// 获得文件Byte数组 /// </summary> /// <param name="filePath">文件路径</param> /// <param name="ifModifiedSince"></param> /// <returns></returns> protected virtual async Task <(FileCacheModel fileCacheModel, bool isCache)> GetFileBytesAsync(string filePath, ICharSequence ifModifiedSince) { while (_readingFiles.Any(m => m.Key.Equals(filePath))) { await Task.Delay(1000); } if (ifModifiedSince != null) { return(await GetFileBytesFromCacheAsync(filePath, ifModifiedSince)); } FileCacheModel fileCacheModel = await GetFileBytesFromFileAsync(filePath); return(fileCacheModel, false); }
/// <summary> /// Checks the service for any existing uploads of an APK. /// This helps to prevent the same APK from being uploaded at the same time. /// </summary> /// <param name="packageDetails">Details of the package being checked for active uploads.</param> public static bool CheckForActiveUpload(PackageDetails packageDetails) { try { CleanUpUploads(); var uploadKey = GetUploadKey(packageDetails); return(activeUploads.Any(x => x.Key.Equals(uploadKey, StringComparison.OrdinalIgnoreCase))); } catch (Exception e) { LoggingControl.Log("Error while checking for uploads for " + packageDetails.Name + " " + packageDetails.Version + ".", e); return(false); } }
/// <summary> /// 是否可以注册 /// </summary> /// <param name="key"></param> /// <param name="model"></param> /// <returns></returns> private (bool canRegister, string errorMessage) CanRegister(string key, RegisterModel model) { try { if (model.Key != ApplicationConfig.ServerInfo.Key) { throw new MateralAPPException("密钥错误"); } if (_registers.ContainsKey(key)) { throw new MateralAPPException("同一连接不可以重复注册"); } if (_registers.Any(m => m.Value.Name.Equals(model.Name, StringComparison.Ordinal))) { throw new MateralAPPException("该服务名称已被注册"); } if (_registers.Any(m => m.Value.Url.Equals(model.Url, StringComparison.Ordinal))) { throw new MateralAPPException("该服务连接已被注册"); } switch (model.ServerCategory) { case ServerCategoryEnum.Authority: case ServerCategoryEnum.ConfigCenter: if (_registers.Any(m => m.Value.ServerCategory == model.ServerCategory)) { throw new MateralAPPException($"{model.ServerCategory.GetDescription()}已被注册"); } break; } return(true, string.Empty); } catch (MateralAPPException exception) { return(false, exception.Message); } }
private void GetPerformanceCounters(string categoryName, string counterName, string instanceName, string seriesName) { var cpuPerfCounter = new PerformanceCounter(categoryName, counterName, instanceName); while (true) { if (!counters.Any(p => p.Key == seriesName)) { var cpuArray = new double[60]; counters[seriesName] = cpuArray; //counters.Add(seriesName, cpuArray); } var localDoubles = counters[seriesName]; localDoubles[localDoubles.Length - 1] = Math.Round(cpuPerfCounter.NextValue(), 0); Array.Copy(localDoubles, 1, localDoubles, 0, localDoubles.Length - 1); //cpuArray[cpuArray.Length - 1] = Math.Round(cpuPerfCounter.NextValue(), 0); //Array.Copy(cpuArray, 1, cpuArray, 0, cpuArray.Length - 1); counters[seriesName] = localDoubles; if (chart1.IsHandleCreated) { this.Invoke(new MethodInvoker(() => this.UpdateCpuChart(seriesName))); } else { //...... } Thread.Sleep(1000); } }
private void GatherData() { var stopwatch = new Stopwatch(); stopwatch.Start(); if (!_subStationDict.Any()) { Log($"采集下无关联传感器."); SleepLifeCycle(); } foreach (var subStationId in _subStationDict.Keys) { stopwatch.Restart(); Log($"采集传感器{subStationId}"); if (!_subStationDict.ContainsKey(subStationId)) { continue; } var now = DateTime.Now; var substationModel = _subStationDict[subStationId]; if (Comm == null || !Comm.IsConnect) { Log($"网络断开,{Comm}重新连接."); Comm?.Start(); Log($"{Comm} 断开, {substationModel.SubStationID}断线"); substationModel.UpdateNetOff(now); FireDasUpdateRealData(substationModel); SleepLifeCycle(null, stopwatch); continue; } var command = SubstationProtocol.GetRealDataCommand(subStationId); if (DasConfig.ShowDetailLogs) Log($"读取[{subStationId}]号传感器实时数据:{command.DataString}"); var result = Comm.Send(command); if (!result) { Log($"读取[{subStationId}]号传感器实时数据失败"); continue; } if (!SpinWait.SpinUntil(() => command.IsReceiveResponse, DasConfig.RecvTimeout * 2)) { Log($"[{subStationId}]号传感器断线"); substationModel.UpdateNetOff(now); FireDasUpdateRealData(substationModel); } SleepLifeCycle(null, stopwatch); Log($"{"=".Repeat(20)}采集传感器[{subStationId}],耗时:{stopwatch.Elapsed.TotalSeconds:F2}秒{"=".Repeat(20)}"); } }
/// <summary> /// 将日子消息如内存队列 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="message"></param> /// <param name="targetObj"></param> private static void Enqueue <T>(T message, string targetObj) { lock (obj) { if (cacheLogQueue.Any(item => item.Key == targetObj)) { cacheLogQueue[targetObj].Enqueue(message); } else { cacheLogQueue[targetObj] = new Queue <object>(); cacheLogQueue[targetObj].Enqueue(message); } } }
internal async Task RemoveAlarmSubscriptionAsync(AlarmSubscription subscription) { var userId = subscription.CallbackHandler.Id; if (_subscriptions.TryRemove(userId, out _)) { if (_alarmIndicationHandler.TryRemove(userId, out _)) { if (!_alarmIndicationHandler.Any()) { await DisableAlarmUpdatesAsync().ConfigureAwait(false); } } } }
/// <inheritdoc cref="ISessionManager"/> public void CloseAll() { UnregisterUoW(); if (!_context.Any()) { Logger.LogDebugFormat("The are no bounded sessions. Nothing to close"); return; } foreach (var session in _context.Values) { CloseSession(session); } }
public void Run() { while (_exception == null && _sheduled.Any(x => !x.Value)) { _sheduled.Where(x => !x.Value && _ready(x.Key)).ForEach(x => Enqueue(x.Key)); _wake.WaitOne(); } while (_exception == null && _tasks.Any()) { _wake.WaitOne(); } Cancel(); }
public static void UnsubscribeFromAllChannels(SPClient client) { client.ReadyForRMI = false; if (_subscribersInfo.Any(s => s.Value.Exists(e => e.Client == client))) { var toRemove = _subscribersInfo.Where(s => s.Value.Where(e => e.Client == client).Any()).ToList(); if (toRemove.Any()) { for (int i = 0; i < toRemove.Count; i++) { _subscribersInfo.Remove(toRemove[i].Key, out _); } } } }
public bool IsExist(int productId) { try { Log.Instance.LogAsInfo($"{nameof(PhotoCache)}.{nameof(IsExist)}: IsExist is starting. {nameof(productId)}:{productId}"); var result = _cache.Any(x => x.Key == productId); Log.Instance.LogAsInfo($"{nameof(PhotoCache)}.{nameof(IsExist)}: IsExist is completed. {nameof(productId)}:{productId}"); return(result); } catch (Exception e) { Log.Instance.ExceptionInfo(e).LogAsError($"{nameof(PhotoCache)}.{nameof(IsExist)}: IsExist Error. {nameof(productId)}:{productId}"); return(false); } }
public async Task <Cacheable> Get(string key) { if (cacheables.Any(a => a.Key == key && !a.Value.IsExpired())) { return(cacheables.FirstOrDefault(f => f.Key == key).Value.Cacheable); } var cacheable = await factory.Create(); var cacheStruct = new CacheStruct(cacheable, policy.GetExpirationDate()); cacheables.AddOrUpdate(key, cacheStruct, (k, oldValue) => oldValue = cacheStruct); return(cacheable); }
protected override async Task TestCoolDown() { if (_runningTasks.Any()) { var lastSize = 0; DateTime lastNewSize = new DateTime(); string testProgressString; while (true) { try { testProgressString = PrintTestProgress(); testProgressString = $"OutStanding: {_totalActiveRequests} {testProgressString}"; if (_totalActiveRequests == 0) { Console.WriteLine(testProgressString); Console.WriteLine("Finished Outstanding Requests"); break; } if (_totalActiveRequests != lastSize) { lastSize = _totalActiveRequests; lastNewSize = DateTime.Now; } else { var secondsSinceLastNewSize = (DateTime.Now - lastNewSize).TotalSeconds; var secondsLeft = TimeSpan.FromMilliseconds(Constants.LoadCoolDownTimeout).TotalSeconds - secondsSinceLastNewSize; Console.WriteLine("No new requests for {0} seconds. Waiting another {1}s to finish", secondsSinceLastNewSize, secondsLeft); if (secondsLeft < 0) { break; } } Console.WriteLine(testProgressString); await Task.Delay(1000); } catch (Exception e) { Console.WriteLine(e); } } } }
private async Task <SyntaxNode> ReduceAsync( Document document, SyntaxNode root, SemanticModel semanticModel, IEnumerable <TextSpan> spans, OptionSet optionSet, IEnumerable <AbstractReducer> reducers, CancellationToken cancellationToken) { // Create a simple interval tree for simplification spans. var spansTree = new SimpleIntervalTree <TextSpan>(TextSpanIntervalIntrospector.Instance, spans); Func <SyntaxNodeOrToken, bool> isNodeOrTokenOutsideSimplifySpans = (nodeOrToken) => !spansTree.GetOverlappingIntervals(nodeOrToken.FullSpan.Start, nodeOrToken.FullSpan.Length).Any(); // Get the list of syntax nodes and tokens that need to be reduced. ImmutableArray <NodeOrTokenToReduce> nodesAndTokensToReduce = this.GetNodesAndTokensToReduce(root, isNodeOrTokenOutsideSimplifySpans); if (nodesAndTokensToReduce.Any()) { if (reducers == null) { reducers = this.GetReducers(); } var reducedNodesMap = new ConcurrentDictionary <SyntaxNode, SyntaxNode>(); var reducedTokensMap = new ConcurrentDictionary <SyntaxToken, SyntaxToken>(); // Reduce all the nodesAndTokensToReduce using the given reducers/rewriters and // store the reduced nodes and/or tokens in the reduced nodes/tokens maps. // Note that this method doesn't update the original syntax tree. await this.ReduceAsync(document, root, nodesAndTokensToReduce, reducers, optionSet, semanticModel, reducedNodesMap, reducedTokensMap, cancellationToken).ConfigureAwait(false); if (reducedNodesMap.Any() || reducedTokensMap.Any()) { // Update the syntax tree with reduced nodes/tokens. root = root.ReplaceSyntax( nodes: reducedNodesMap.Keys, computeReplacementNode: (o, n) => TransformReducedNode(reducedNodesMap[o], n), tokens: reducedTokensMap.Keys, computeReplacementToken: (o, n) => reducedTokensMap[o], trivia: SpecializedCollections.EmptyEnumerable <SyntaxTrivia>(), computeReplacementTrivia: null); } } return(root); }
public string Output(bool printTime = false) { var wr = new StringWriter(); if (runs.Any()) { wr.WriteLine("Cached verification result injector statistics as CSV:"); wr.WriteLine("Request ID, Transformed, Low, Medium, High, Skipped{0}", printTime ? ", Time (ms)" : ""); foreach (var kv in runs.OrderBy(kv => ExecutionEngine.AutoRequestId(kv.Key))) { var t = printTime ? string.Format(", {0,8:F0}", kv.Value.End.Subtract(kv.Value.Start).TotalMilliseconds) : ""; wr.WriteLine("{0,-19}, {1,3}, {2,3}, {3,3}, {4,3}, {5,3}{6}", kv.Key, kv.Value.TransformedImplementationCount, kv.Value.LowPriorityImplementationCount, kv.Value.MediumPriorityImplementationCount, kv.Value.HighPriorityImplementationCount, kv.Value.SkippedImplementationCount, t); } } return(wr.ToString()); }
internal static async Task <bool> StopTimeshiftAsync(IOwinContext context, int id, string userName) { if (!CLIENT_CHANNELS.TryRemove(userName, out ChannelInfo channel)) { return(false); } if (channel != null && !CLIENT_CHANNELS.Any(c => c.Value?.ChannelId == channel.ChannelId)) { if (!(await TimeshiftControl.StopTimeshiftAsync(channel.UserName, channel.SlotIndex).ConfigureAwait(false))) { return(false); } } return(true); }
public async Task HashSetFieldsAsync(string key, ConcurrentDictionary <string, string> fields) { if (fields == null || !fields.Any()) { return; } var hs = await HashGetAsync(key); foreach (var field in fields) { hs[field.Key] = field.Value; } await HashSetAsync(key, hs); }
public int GetAveragePizzaTimes() { if (!_pizzaTimes.Any()) { return(10); } var cookedPizzas = _pizzaTimes.Values.Where(pi => pi.PizzaCooked.HasValue).ToArray(); if (!cookedPizzas.Any()) { return(10); } return((int)cookedPizzas.Select(pi => (pi.PizzaCooked.Value - pi.OrderRecieved).TotalMinutes).Average()); }
public int Login(string userName) { if (ConnectedClients.Any(client => String.Equals(client.Key, userName, StringComparison.CurrentCultureIgnoreCase))) { Console.WriteLine("[{0}] user: {1} is already logged in", DateTime.Now, userName); return(1); } var estabilishedUserConnection = OperationContext.Current.GetCallbackChannel <IClient>(); var newClient = new ConnectedClient { Connection = estabilishedUserConnection, UserName = userName }; ConnectedClients.TryAdd(userName, newClient); Console.WriteLine("[{0}] user: {1} logged in!", DateTime.Now, userName); return(0); }
private void InitTasksRegistry() { Dictionary <string, JobModel> taskModels = Exports.ToDictionary(exports => exports.GetType().Assembly.FullName, exports => { var taskContainer = CreateTaskContainer(exports); var panteonTask = taskContainer.Resolve <IPanteonWorker>(); return(new JobModel { Task = panteonTask, Container = taskContainer }); }); if (_jobModelRegistry.Any()) { foreach (KeyValuePair <string, JobModel> pair in taskModels) { if (!_jobModelRegistry.ContainsKey(pair.Key)) { JobModel jobModel = _jobModelRegistry.AddOrUpdate(pair.Key, pair.Value, (s, model) => /*TODO: change detection*/ pair.Value.Container.GetHashCode() != model.Container.GetHashCode() ? pair.Value : model); jobModel?.Task?.Init(autoRun: true); } else { //TODO,: UPDATE or REMOVE //TaskModel taskModel; //if (_taskModelDictionary.TryRemove(pair.Key, out taskModel)) //{ // StopTask(taskModel.Task.Name); //} } } } else { _jobModelRegistry = new ConcurrentDictionary <string, JobModel>(taskModels); } }
protected override void Dispose(bool disposing) { if (!disposing || Disposed) { base.Dispose(disposing); return; } if (_items.NotAnySafe()) { base.Dispose(disposing); return; } while (_items.Any()) { KeyValuePair <ManagedIDisposableKey, IDisposable> holder = _items.FirstOrDefault(); ManagedIDisposableKey key = holder.Key; if (key.Id == Guid.Empty) { continue; } // _items[holder.Key] = null; int attempt = 0; IDisposable value; while (!_items.TryRemove(holder.Key, out value)) { Thread.Sleep(1); attempt++; if (attempt >= retryLimit) { throw new InvalidOperationException("Unable to free resource - " + (value?.GetType()?.FullName ?? "unknown IDisposable")); } } ; key?.TriggerCallbefore(value, this); try { value?.Dispose(); } catch (ObjectDisposedException) { // If it's already disposed that is ok } key?.TriggerCallback(this); } base.Dispose(disposing); }
public static async Task AddChannel(IHubClients <IOrchestrationClient> clients, string channelname, ILogger logger) { logger.LogInformation($"AddChannel: Channel {channelname}"); if (ConnectionChannelList.Any()) { if (!ConnectionChannelList.Values.SelectMany(_ => _).Contains(channelname, StringComparer.OrdinalIgnoreCase)) { var clientToConnect = ConnectionChannelList.OrderByDescending(_ => _.Value.Count).First(); clientToConnect.Value.Add(channelname); await clients.Client(clientToConnect.Key).JoinChannelsCommand(clientToConnect.Value); } } else { logger.LogInformation("No active bots connected"); } }
private void InstallDependencies() { viewMode = ViewMode.Installing; PluginLog.Log("Installing dependencies..."); var installTasks = missingDependencies.Select(InstallDependency); Task.WhenAll(installTasks).ContinueWith(task => { var failed = installProgress.Any(pair => pair.Value == DEP_FAILED); viewMode = failed ? ViewMode.Failed : ViewMode.Complete; PluginLog.Log($"Dependency install {viewMode}."); try { Directory.Delete(Path.Combine(dependencyDir, DOWNLOAD_DIR), true); } catch { } }); }
public void Connect(string name) { if (connectedClients.Any(c => c.Value.ToLower() == name.ToLower())) { string reason = $"the {name} is already taken, please choose another name"; Clients.Caller.rejected(reason); } else { var connectionId = this.Context.ConnectionId; connectedClients.AddOrUpdate(connectionId, name, (key, value) => { return(value); }); Clients.Caller.connected(); Clients.Others.newClient(name); Console.WriteLine("new client connected: " + name); Console.WriteLine("Total Clients: " + connectedClients.Count()); } }
public async Task End() { if (!_votes.Any()) { await _channel.SendErrorAsync(GetText("acrophobia"), GetText("acro_no_votes_cast")).ConfigureAwait(false); return; } var table = _votes.OrderByDescending(v => v.Value); var winner = table.First(); var embed = new EmbedBuilder().WithOkColor() .WithTitle(GetText("acrophobia")) .WithDescription(GetText("acro_winner", Format.Bold(_submissions[winner.Key].ToString()), Format.Bold(winner.Value.ToString()))) .WithFooter(efb => efb.WithText(winner.Key.ToLowerInvariant().ToTitleCase())); await _channel.EmbedAsync(embed).ConfigureAwait(false); }
public void RaiseNewPCEntries(ConcurrentDictionary<UInt32, ActorEntity> actorEntities) { if (!actorEntities.Any()) { return; } // THIRD PARTY PluginHost.Instance.RaiseNewPCEntries(actorEntities); }
private static bool RunDictionaryTest(int cLevel, int initSize, int threads, int addsPerThread, TestMethod testMethod) { TestHarness.TestLog("* RunDictionaryTest_{0}, Level={1}, initSize={2}, threads={3}, addsPerThread={4})", PrintTestMethod(testMethod), cLevel, initSize, threads, addsPerThread); ConcurrentDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1); int count = threads; using (ManualResetEvent mre = new ManualResetEvent(false)) { for (int i = 0; i < threads; i++) { int ii = i; ThreadPool.QueueUserWorkItem( (o) => { for (int j = 0; j < addsPerThread; j++) { //call either of the two overloads of GetOrAdd if (j + ii % 2 == 0) { dict.GetOrAdd(j, -j); } else { dict.GetOrAdd(j, x => -x); } } if (Interlocked.Decrement(ref count) == 0) mre.Set(); }); } mre.WaitOne(); } bool passed = true; if (dict.Any(pair => pair.Key != -pair.Value)) { TestHarness.TestLog(" > Invalid value for some key in the dictionary."); passed = false; } var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray(); var expectKeys = Enumerable.Range(0, addsPerThread); if (!gotKeys.SequenceEqual(expectKeys)) { TestHarness.TestLog(" > The set of keys in the dictionary is invalid."); passed = false; } // Finally, let's verify that the count is reported correctly. int expectedCount = addsPerThread; int count1 = dict.Count, count2 = dict.ToArray().Length, count3 = dict.ToList().Count; if (count1 != expectedCount || count2 != expectedCount || count3 != expectedCount) { TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary. Expected {0}, Dict.Count {1}, ToArray.Length {2}, ToList.Count {3}", expectedCount, count1, count2, count3); passed = false; } return passed; }
public static void TryToAddModels(ConcurrentDictionary<string, TypeInfo> models, Type type, XmlCommentDocumentationProvider docProvider, ConcurrentDictionary<string, string> typesToReturn = null, int level = 0) { var _type = type; if (type.IsArray) _type = type.GetElementType(); else if (type.IsGenericType) _type = type.GetGenericArguments().First(); string typeName = GetTypeName(_type); if (models.Any(m => m.Key == typeName)) return; if (IsOutputable(_type)) { var typeInfo = new TypeInfo { id = typeName }; if (!IgnoreTypes.Contains(_type.Name.ToLower())) { typeInfo.description = docProvider.GetSummary(_type); } //Ignore properties for .net types if (!_type.Assembly.FullName.Contains("System") && !_type.Assembly.FullName.Contains("mscorlib")) { var modelInfoDic = new Dictionary<string, PropInfo>(); foreach (var propertyInfo in _type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) { if (propertyInfo.GetCustomAttributes(typeof(JsonIgnoreAttribute), false).FirstOrDefault() != null) continue; var propInfo = new PropInfo(); string propName = GetPropertyName(propertyInfo); Type propType = docProvider.GetType(propertyInfo); SwaggerType swaggerType = GetSwaggerType(propType); propInfo.type = swaggerType.Name; propInfo.items = swaggerType.Items; propInfo.required = IsRequired(propertyInfo, docProvider); if (!modelInfoDic.Keys.Contains(propName)) modelInfoDic.Add(propName, propInfo); if (!IgnoreTypes.Contains(propInfo.type)) { propInfo.description = docProvider.GetSummary(propertyInfo); if (propertyInfo.PropertyType.IsEnum) { modelInfoDic[propName].@enum = propertyInfo.PropertyType.GetEnumNames(); } //Don't go too deep if (level < 10) { TryToAddModels(models, swaggerType.type, docProvider, typesToReturn, ++level); } } } typeInfo.properties = modelInfoDic; } if (_type.IsEnum) { typeInfo.values = _type.GetEnumNames(); } models.TryAdd(typeName, typeInfo); } }
public void WhenInboundHasProblemsOutboundJustKeepsOnTrucking(int numberOfMessages) { // arrange var receivedMessageCount = 0; var messageTracker = new ConcurrentDictionary<Guid, int>(); var resetEvent = new ManualResetEvent(false); orderSystemHandlerActivator.Handle<PlaceOrderRequest>(req => { if (req.What != "beer" || req.HowMuch != 12) return; OnCommit.Do(() => { messageTracker.AddOrUpdate(req.MsgId, 1, (id, count) => count + 1); var newValue = Interlocked.Increment(ref receivedMessageCount); if (newValue >= numberOfMessages) { resetEvent.Set(); } }); }); var timeout = numberOfMessages.Seconds(); var keepMakingChaos = true; var chaosMonkey = new Thread(() => { while (keepMakingChaos) { Thread.Sleep(0.1.Seconds()); inbound.Stop(); Console.WriteLine("Inbound stopped - {0} messages processed...", receivedMessageCount); Thread.Sleep(0.2.Seconds()); inbound.Start(); Thread.Sleep(2.2331.Seconds()); } }); // act chaosMonkey.Start(); numberOfMessages.Times(() => pricedesk.Send(CreateMessage())); // assert var resetEventWasSet = resetEvent.WaitOne(timeout + 5.Seconds()); keepMakingChaos = false; chaosMonkey.Join(); // chill, be more sure to empty the queue completely Thread.Sleep(1.Seconds()); Assert.That(resetEventWasSet, Is.True, "Request was not received in order system within timeout of {0}", timeout); receivedMessageCount.ShouldBeGreaterThanOrEqualTo(numberOfMessages); Console.WriteLine("Actual number of received messages: {0}", receivedMessageCount); if (messageTracker.Any(t => t.Value > 1)) { Console.WriteLine(@"The following IDs were received more than once: {0}", string.Join(Environment.NewLine, messageTracker.Where(t => t.Value > 1).Select(kvp => " " + kvp.Key + ": " + kvp.Value))); } messageTracker.Count.ShouldBe(numberOfMessages); }
public BermudaResult GetData(string domain, string query, string mapreduce, string merge, string paging, int remdepth, string command, string cursor, string paging2) { var args = ParseCommand(command); bool noCache = args.Contains("-nocache"); bool makeCursor = cursor == MakeCursorToken; bool useCursor = !makeCursor && !string.IsNullOrWhiteSpace(cursor); DateTime minDate = DateTime.MinValue; DateTime maxDate = DateTime.MaxValue; if (remdepth > 0) { //map var queryHash = cursor ?? GetQueryHash(domain, query, mapreduce, merge, paging, null); //reduce BermudaResult cachedDatapoints; if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime) { #if DEBUG if (CacheTraceMessageLevel < 3) Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]"); #endif if (useCursor) { var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType); return GetCursorData(paging2, cachedDatapoints, dataType); } else { return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_1" }, CacheKey = cachedDatapoints.CacheKey }; } } else { if (useCursor) throw new Exception("Cursor " + cursor + " not found"); //var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true); var reducers = HostEnvironment.Instance.GetAvailablePeerConnections(); if (!reducers.Any()) throw new Exception("Specified dataset not loaded: " + domain); ConcurrentDictionary<PeerInfo, BermudaResult> results = new ConcurrentDictionary<PeerInfo, BermudaResult>(); Stopwatch sw = new Stopwatch(); sw.Start(); List<Task> tasks = new List<Task>(); foreach (var reducer in reducers) { Task t = new Task((peerObj) => { var peerInfo = peerObj as PeerInfo; var initiated = DateTime.Now; var subqueryHash = GetQueryHash(domain, query, mapreduce, merge, paging, peerInfo.ToString()); Stopwatch sw3 = new Stopwatch(); sw3.Start(); //see if the cache contains a matching result and return it if it's not outdated BermudaResult cachedDatapoints2; if (!noCache && CachedData.TryGetValue(subqueryHash, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime) { if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]"); BermudaResult res = null; if (useCursor) { var dataType2 = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType); res = GetCursorData(paging2, cachedDatapoints2, dataType2); } else { res = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_2" } }; } results[peerInfo] = res; } else { try { Stopwatch sw2 = new Stopwatch(); sw2.Start(); BermudaResult subresult = null; if (peerInfo.Equals(Endpoint)) { subresult = GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2); } else { using (var client = HostEnvironment.GetServiceClient(peerInfo)) { subresult = client.GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2); } //subresult = GetDataFromPeer(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, command, assignment.PeerEndpoint.Endpoint); } sw2.Stop(); subresult.CreatedOn = DateTime.Now.Ticks; subresult.Metadata.Initiated = initiated; subresult.Metadata.Completed = DateTime.Now; subresult.Metadata.OperationTime = sw2.Elapsed; results[peerInfo] = CachedData[subqueryHash] = subresult; } catch (Exception ex) { results[peerInfo] = new BermudaResult { Error = "[Failed Node] " + ex }; } } }, reducer, TaskCreationOptions.LongRunning); tasks.Add(t); t.Start(); } Task.WaitAll(tasks.ToArray()); sw.Stop(); #if DEBUG Trace.WriteLine("Join Time:" + sw.Elapsed); #endif if (results.Any(x => x.Value.Error != null)) throw new BermudaException("Some nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error))); if (results.All(x => x.Value.Data == null)) return new BermudaResult { Metadata = new BermudaNodeStatistic { Notes = "No Data" } }; //if all results are not the same time throw an error if (results.GroupBy(x => x.Value.DataType).Count() > 1) throw new BermudaException("Subresults must all return the same type"); var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null); if (dataTypeDescriptor == null) return new BermudaResult { Error = "Could not determine the merge type, none of the nodes provided type info" }; //use the passed combine espression to make multiple datapoint sets into one var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor); //allItems = results.Values.SelectMany(x => x.DataObject) var totalJson = "[" + string.Join(",", results.Values.Where(x => !string.IsNullOrWhiteSpace(x.Data)).Select(x => x.Data.Trim('[', ']')).Where(x => !string.IsNullOrWhiteSpace(x))) + "]"; var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true); //var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson); //var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count(); //InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType); var mergeFunc = GetMergeFunc(merge, mapreduce, dataType, dataType); if (mergeFunc != null) { //var dataType = "kdsajkdsa"; var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke"); allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc); } var pagingFunc = GetPagingFunc(paging, dataType); if (pagingFunc != null) { var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke"); allItems = pagingInvokeMethod.Invoke(pagingFunc, new object[] { allItems }); } //figure out the metadata var finalMetadata = new BermudaNodeStatistic { Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = HostEnvironment.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.Metadata).ToArray() }; var arraylol = ToArrayCollection(allItems, dataType); var json = JsonConvert.SerializeObject(arraylol); //var json = JsonConvert.SerializeObject(allItems); var originalData = makeCursor ? arraylol : null; var finalResult = new BermudaResult { DataType = dataTypeDescriptor, OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = finalMetadata, CacheKey = queryHash }; CachedData[queryHash] = finalResult; return finalResult; } } else { ConcurrentDictionary<string, BermudaResult> results = new ConcurrentDictionary<string, BermudaResult>(); BermudaNodeStatistic stats = new BermudaNodeStatistic(); var bucketInterfaces = HostEnvironment.Instance.GetBucketInterfacesForDomain(domain); if (!bucketInterfaces.Any()) throw new BermudaException("Data not loaded for: " + domain); if (bucketInterfaces.Count() > 1) throw new BermudaException("Multiple buckets not supported by BermudaMapReduce"); var queryHash = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString()); BermudaResult cachedDatapoints; if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime) { if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]"); if (useCursor) { var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType); return GetCursorData(paging2, cachedDatapoints, dataType); } else { return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_3" }, CacheKey = queryHash }; } } else { //Chad: short circuiting to test WCF response time in Azure //return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() }; //IEnumerable<Datapoint> datapoints = null; object datapoints = null; Stopwatch sw = new Stopwatch(); sw.Start(); Type itemType = null; Type resultType = null; string json = null; foreach (var bucketInterface in bucketInterfaces) { var bucketKey = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString()); //see if the cache contains a matching result and return it if it's not outdated BermudaResult cachedDatapoints2; if (!noCache && CachedData.TryGetValue(bucketKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime) { if (CacheTraceMessageLevel < 1) Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]"); if (useCursor) { if (cachedDatapoints2.OriginalData == null) throw new Exception("Cursor " + cursor + " contains null data"); var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType); results[bucketInterface.Name] = GetCursorData(paging2, cachedDatapoints2, dataType); } else { results[bucketInterface.Name] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_4" } }; json = cachedDatapoints2.Data; } } else { //get mentions var collections = GetCollections(query, mapreduce); if (collections.Count() > 1) throw new BermudaException("More than one collection specified: " + string.Join(",", collections)); var table = collections.FirstOrDefault(); var tableName = table == null ? null : table.Source; var raw = bucketInterface.GetData(tableName); //var rawType = raw.GetType(); //itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType); itemType = bucketInterface.GetDataType(tableName); var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType); var queryFunc = GetFilterFunc(query, itemType); var pagingFunc = GetPagingFunc(paging, resultType); var minDateTicks = minDate.Ticks; var maxDateTicks = maxDate.Ticks; object subresult = raw; //queryFunc == null ? // raw.AsParallel() : //minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ? // raw.AsParallel().Where(x => queryFunc) : // raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters)); if (json == null) { if (queryFunc != null) { var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke"); subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult }); } //reduce them using the passed expression if (mapreduceFunc != null) { var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke"); subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult }); } if (pagingFunc != null) { var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke"); subresult = pagingInvokeMethod.Invoke(pagingFunc, new object[] { subresult }); } datapoints = subresult; } //format a metada string if (!args.Contains("-nocount")) { stats.TotalItems = bucketInterface.GetCount(tableName); //stats.FilteredItems = filtered.Count(); //stats.ReducedItems = subresult.Count(); } //cache the result //results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() }; //CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() }; } } //figure out the metadata //var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata)); stats.NodeId = HostEnvironment.Instance.CurrentInstanceId; stats.Notes = "Computed Datapoints"; //Trace.WriteLine("total mentions processed: " + mentionCount); //var datapoints = results.Values.SelectMany(x => x.Datapoints); if (datapoints == null) return new BermudaResult() { Metadata = new BermudaNodeStatistic { Notes = "No Results" } }; //foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count; var mergeFunc = GetMergeFunc(merge, mapreduce, itemType, resultType); if (mergeFunc != null) { var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke"); datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints }); } stats.LinqExecutionTime = sw.Elapsed; var arraylol = ToArrayCollection(datapoints, resultType); if (json == null && datapoints != null) { json = JsonConvert.SerializeObject(arraylol); } //var json = JsonConvert.SerializeObject(datapoints); var originalData = makeCursor ? arraylol : null; var result = CachedData[queryHash] = new BermudaResult { DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = stats }; sw.Stop(); return result; } } }
public TaskView(Gtk.TreeModel model, IPreferences preferences) { if (preferences == null) throw new ArgumentNullException ("preferences"); TreeView = new TreeView (); #if GTK_2_12 // set up the timing for the tooltips TreeView.Settings.SetLongProperty("gtk-tooltip-browse-mode-timeout", 0, "Tasque:TaskTreeView"); TreeView.Settings.SetLongProperty("gtk-tooltip-browse-timeout", 750, "Tasque:TaskTreeView"); TreeView.Settings.SetLongProperty("gtk-tooltip-timeout", 750, "Tasque:TaskTreeView"); ConnectEvents(); #endif // TODO: Modify the behavior of the TreeView so that it doesn't show // the highlighted row. Then, also tie in with the mouse hovering // so that as you hover the mouse around, it will automatically // select the row that the mouse is hovered over. By doing this, // we should be able to not require the user to click on a task // to select it and THEN have to click on the column item they want // to modify. filterTaskList = null; modelFilter = new Gtk.TreeModelFilter (model, null); modelFilter.VisibleFunc = FilterFunc; modelFilter.RowInserted += OnRowInsertedHandler; modelFilter.RowDeleted += OnRowDeletedHandler; Refilter (); //Model = modelFilter TreeView.Selection.Mode = Gtk.SelectionMode.Single; TreeView.RulesHint = false; TreeView.HeadersVisible = false; TreeView.HoverSelection = true; // TODO: Figure out how to turn off selection highlight columns = new List<ITaskColumn> (); var nodeList = AddinManager.GetExtensionNodes (typeof(ITaskColumn)).Cast<TaskColNode> (); var nodes = new List<TaskColNode> (nodeList); foreach (var node in nodes) AddColumn (node, nodes); rowEditingDictionary = new ConcurrentDictionary<ITaskColumn, TaskRowEditingEventArgs> (); columns.Sort ((x, y) => x.DefaultPosition.CompareTo (y.DefaultPosition)); foreach (var col in columns) { col.Initialize (Model, this, preferences); col.CellEditingStarted += (sender, e) => { if (rowEditingDictionary.IsEmpty) IsTaskBeingEdited = true; if (!rowEditingDictionary.Any (v => v.Value.ITask == e.ITask)) { if (RowEditingStarted != null) RowEditingStarted (this, e); } rowEditingDictionary.TryAdd ((ITaskColumn)sender, e); }; col.CellEditingFinished += (sender, e) => { TaskRowEditingEventArgs args; rowEditingDictionary.TryRemove ((ITaskColumn)sender, out args); if (!rowEditingDictionary.Any (v => v.Value.ITask == e.ITask)) { if (RowEditingFinished != null) RowEditingFinished (this, e); } if (rowEditingDictionary.IsEmpty) IsTaskBeingEdited = false; }; TreeView.AppendColumn (col.TreeViewColumn); } }
private async Task RemoveObjectsFromTenantAsync() { var client = TestClients.GetSAuthc1Client(); var results = new ConcurrentDictionary<string, Exception>(); // Delete applications var deleteApplicationTasks = this.CreatedApplicationHrefs.Select(async href => { try { var application = await client.GetResourceAsync<IApplication>(href); var deleteResult = await application.DeleteAsync(); results.TryAdd(href, null); } catch (ResourceException rex) { if (rex.Code == 404) { // Already deleted results.TryAdd(href, null); } } catch (Exception e) { results.TryAdd(href, e); } }); // Delete directories var deleteDirectoryTasks = this.CreatedDirectoryHrefs.Select(async href => { try { var directory = await client.GetResourceAsync<IDirectory>(href); var deleteResult = await directory.DeleteAsync(); results.TryAdd(href, null); } catch (ResourceException rex) { if (rex.Code == 404) { // Already deleted results.TryAdd(href, null); } } catch (Exception e) { results.TryAdd(href, e); } }); // Delete organizations var deleteOrganizationTasks = this.CreatedOrganizationHrefs.Select(async href => { try { var org = await client.GetResourceAsync<IOrganization>(href); var deleteResult = await org.DeleteAsync(); results.TryAdd(href, null); } catch (ResourceException rex) { if (rex.Code == 404) { // Already deleted results.TryAdd(href, null); } } catch (Exception e) { results.TryAdd(href, e); } }); await Task.WhenAll( Task.WhenAll(deleteApplicationTasks), Task.WhenAll(deleteDirectoryTasks), Task.WhenAll(deleteOrganizationTasks)); // All done! Throw errors if any occurred bool anyErrors = results.Any(kvp => kvp.Value != null); if (anyErrors) { throw new ApplicationException( "Errors occurred during test cleanup. Full log: " + Environment.NewLine + string.Join(Environment.NewLine, results.Select(kvp => $"{kvp.Key} : '{(kvp.Value == null ? "Good" : kvp.Value.Message)}'"))); } }
internal static void Configure() { if(null != All) { return; } lock(_Lock){ if(null != All) { return; } // all All = new ConcurrentDictionary <string, IDataProviderSetting>(); IEnumerator all = ConfigurationManager .ConnectionStrings .GetEnumerator(); while(all.MoveNext()){ var settings = all.Current as ConnectionStringSettings; if(null == settings) { continue; } string[] names = settings.Name.SplitAndTrim(","); foreach (var name in names) { Add(new DataProviderSetting() { Name = name.ToUpper(), ConnectionString = settings.ConnectionString, ProviderName = settings.ProviderName, }); } } if(All.Any() && null == Default) { Default = All.FirstOrDefault().Value; } } }
private static bool RunDictionaryTest_Add1(int cLevel, int initSize, int threads, int addsPerThread) { TestHarness.TestLog( "* RunDictionaryTest_Add1(cLevel={0}, initSize={1}, threads={2}, addsPerThread={3})", cLevel, initSize, threads, addsPerThread); IDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1); int count = threads; using (ManualResetEvent mre = new ManualResetEvent(false)) { for (int i = 0; i < threads; i++) { int ii = i; ThreadPool.QueueUserWorkItem( (o) => { for (int j = 0; j < addsPerThread; j++) { dict.Add(j + ii * addsPerThread, -(j + ii * addsPerThread)); } if (Interlocked.Decrement(ref count) == 0) mre.Set(); }); } mre.WaitOne(); } if (dict.Any(pair => pair.Key != -pair.Value)) { TestHarness.TestLog(" > Invalid value for some key in the dictionary."); return false; } var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray(); var expectKeys = Enumerable.Range(0, threads * addsPerThread); if (!gotKeys.SequenceEqual(expectKeys)) { TestHarness.TestLog(" > The set of keys in the dictionary is invalid."); return false; } // Finally, let's verify that the count is reported correctly. int expectedCount = threads * addsPerThread; if (dict.Count != expectedCount || dict.ToArray().Length != expectedCount || dict.ToList().Count() != expectedCount) { TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary."); return false; } return true; }
private async Task<ConcurrentDictionary<Document, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>>> CreateDocumentMapAsync( ConcurrentDictionary<Project, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>> projectMap) { using (Logger.LogBlock(FeatureId.FindReference, FunctionId.FindReference_CreateDocumentMapAsync, this.cancellationToken)) { Func<Document, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>> createQueue = d => new ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>(); var documentMap = new ConcurrentDictionary<Document, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>>(); #if PARALLEL Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () => { projectMap.AsParallel().WithCancellation(cancellationToken).ForAll(kvp => { var project = kvp.Key; var projectQueue = kvp.Value; projectQueue.AsParallel().WithCancellation(cancellationToken).ForAll(symbolAndFinder => { var symbol = symbolAndFinder.Item1; var finder = symbolAndFinder.Item2; var documents = finder.DetermineDocumentsToSearch(symbol, project, cancellationToken) ?? SpecializedCollections.EmptyEnumerable<Document>(); foreach (var document in documents.Distinct().WhereNotNull()) { if (includeDocument(document)) { documentMap.GetOrAdd(document, createQueue).Enqueue(symbolAndFinder); } } }); }); }); #else foreach (var kvp in projectMap) { var project = kvp.Key; var projectQueue = kvp.Value; foreach (var symbolAndFinder in projectQueue) { this.cancellationToken.ThrowIfCancellationRequested(); var symbol = symbolAndFinder.Item1; var finder = symbolAndFinder.Item2; var documents = await finder.DetermineDocumentsToSearchAsync(symbol, project, this.documents, cancellationToken).ConfigureAwait(false) ?? SpecializedCollections.EmptyEnumerable<Document>(); foreach (var document in documents.Distinct().WhereNotNull()) { if (this.documents == null || this.documents.Contains(document)) { documentMap.GetOrAdd(document, createQueue).Enqueue(symbolAndFinder); } } } } #endif Contract.ThrowIfTrue(documentMap.Any(kvp => kvp.Value.Count != kvp.Value.ToSet().Count)); return documentMap; } }
private async Task<ConcurrentDictionary<Project, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>>> CreateProjectMapAsync( ConcurrentSet<ISymbol> symbols) { using (Logger.LogBlock(FeatureId.FindReference, FunctionId.FindReference_CreateProjectMapAsync, this.cancellationToken)) { Func<Project, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>> createQueue = p => new ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>(); var projectMap = new ConcurrentDictionary<Project, ConcurrentQueue<ValueTuple<ISymbol, IReferenceFinder>>>(); #if PARALLEL Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () => { symbols.AsParallel().WithCancellation(cancellationToken).ForAll(s => { finders.AsParallel().WithCancellation(cancellationToken).ForAll(f => { var projects = f.DetermineProjectsToSearch(s, solution, cancellationToken) ?? SpecializedCollections.EmptyEnumerable<Project>(); foreach (var project in projects.Distinct()) { projectMap.GetOrAdd(project, createQueue).Enqueue(ValueTuple.Create(s, f)); } }); }); }); #else var scope = this.documents != null ? this.documents.Select(d => d.Project).ToImmutableHashSet() : null; foreach (var s in symbols) { foreach (var f in finders) { this.cancellationToken.ThrowIfCancellationRequested(); var projects = await f.DetermineProjectsToSearchAsync(s, solution, scope, cancellationToken).ConfigureAwait(false) ?? SpecializedCollections.EmptyEnumerable<Project>(); foreach (var project in projects.Distinct().WhereNotNull()) { if (scope == null || scope.Contains(project)) { projectMap.GetOrAdd(project, createQueue).Enqueue(ValueTuple.Create(s, f)); } } } } #endif Contract.ThrowIfTrue(projectMap.Any(kvp => kvp.Value.Count != kvp.Value.ToSet().Count)); return projectMap; } }
// TODO: VehicleDetails will have to be changed to take an IVehicle as the value data type // TODO: Pass in Enum of vehicle type to use for the processing call public void CrawlWikiSectionPagesForData(HtmlDocument wikiHomePage, ConcurrentDictionary<string, HtmlDocument> vehicleWikiPagesContent, ConcurrentDictionary<string, string> localFileChanges, Dictionary<string, GroundVehicle> vehicleDetails, List<string> errorsList, Stopwatch overallStopwatch, bool createJsonFiles, bool createHtmlFiles, bool createExcelFile) { bool parseErrorsEncountered = _webCrawler.DoesTheDocumentContainParseErrors(wikiHomePage); if (parseErrorsEncountered) { _consoleManager.HandleHtmlParseErrors(wikiHomePage); } else { // Setup initial vars List<HtmlNode> vehicleWikiEntryLinks = new List<HtmlNode>(); webCrawlerStopwatch.Start(); // This is outside of the method because of the recursive call and we don't want the user having to press enter more than once _consoleManager.WriteInputInstructionsAndAwaitUserInput(ConsoleColor.Yellow, ConsoleKey.Enter, "Press ENTER to begin searching for links to vehicle pages."); Dictionary<string, int> linksFound = _webCrawler.GetLinksToVehiclePages(vehicleWikiEntryLinks, wikiHomePage); int totalNumberOfLinksBasedOnPageText = linksFound.Where(l => l.Key.Equals("TotalNumberOfLinksBasedOnPageText")).Single().Value; int totalNumberOfLinksFoundViaDomTraversal = linksFound.Where(l => l.Key.Equals("TotalNumberOfLinksFoundViaDomTraversal")).Single().Value; webCrawlerStopwatch.Stop(); // Setup thread-safe collections for processing ConcurrentDictionary<int, HtmlNode> linksToVehicleWikiPages = new ConcurrentDictionary<int, HtmlNode>(); // Populate the full list of links we need to traverse for (int i = 0; i < vehicleWikiEntryLinks.Count(); i++) { HtmlNode linkNode = vehicleWikiEntryLinks[i]; linksToVehicleWikiPages.TryAdd(i, linkNode); } pageHtmlRetrievalStopwatch.Start(); // Crawl the pages concurrently Task[] webCrawlerTasks = new Task[4] { // Going from 2 to 4 tasks halves the processing time, after 4 tasks the performance gain is negligible Task.Factory.StartNew(() => _webCrawler.GetPageHtml(linksToVehicleWikiPages, vehicleWikiPagesContent)), Task.Factory.StartNew(() => _webCrawler.GetPageHtml(linksToVehicleWikiPages, vehicleWikiPagesContent)), Task.Factory.StartNew(() => _webCrawler.GetPageHtml(linksToVehicleWikiPages, vehicleWikiPagesContent)), Task.Factory.StartNew(() => _webCrawler.GetPageHtml(linksToVehicleWikiPages, vehicleWikiPagesContent)) }; // Wait until we have crawled all of the pages Task.WaitAll(webCrawlerTasks); _consoleManager.WritePaddedText("Finished extracting html documents from vehicle pages."); pageHtmlRetrievalStopwatch.Stop(); _consoleManager.WriteHorizontalSeparator(); _consoleManager.HandleCreateFileTypePrompts(createJsonFiles, createHtmlFiles, createExcelFile); int indexPosition = 1; processingStopwatch.Start(); // Extract information from the pages we've traversed ProcessGroundForcesWikiHtmlFiles(vehicleWikiPagesContent, localFileChanges, vehicleDetails, vehicleWikiEntryLinks, errorsList, indexPosition, totalNumberOfLinksBasedOnPageText, createJsonFiles, createHtmlFiles, createExcelFile); processingStopwatch.Stop(); _consoleManager.WriteLineInColourPreceededByBlankLine(ConsoleColor.Green, $"Finished processing html files for vehicle data{(createExcelFile || createHtmlFiles || createJsonFiles ? " and writing local changes." : ".")}"); if (localFileChanges.Any()) { _logger.HandleLocalFileChanges(localFileChanges); } _consoleManager.WriteHorizontalSeparator(); if (errorsList.Any()) { _logger.HandleProcessingErrors(errorsList); } _consoleManager.WriteHorizontalSeparator(); overallStopwatch.Stop(); _consoleManager.WriteProcessingSummary(overallStopwatch.Elapsed, totalNumberOfLinksBasedOnPageText, totalNumberOfLinksFoundViaDomTraversal, vehicleDetails.Count()); } }