public async Task RunawayThreads() { using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { try { var threadsUsage = new ThreadsUsage(); // need to wait to get a correct measure of the cpu await Task.Delay(100); var result = threadsUsage.Calculate(); context.Write(writer, new DynamicJsonValue { ["Runaway Threads"] = result.ToJson() }); } catch (Exception e) { context.Write(writer, new DynamicJsonValue { ["Error"] = e.ToString() }); } writer.Flush(); } } }
protected override async Task DoWork() { var now = DateTime.UtcNow; var timeSpan = now - _lastSentNotification; if (timeSpan < _notificationsThrottle) { await WaitOrThrowOperationCanceled(_notificationsThrottle - timeSpan); } try { if (CancellationToken.IsCancellationRequested) { return; } if (_watchers.Count == 0) { return; } var threadsInfo = _threadsUsage.Calculate(); foreach (var watcher in _watchers) { // serialize to avoid race conditions // please notice we call ToJson inside a loop since DynamicJsonValue is not thread-safe watcher.NotificationsQueue.Enqueue(threadsInfo.ToJson()); } } finally { _lastSentNotification = DateTime.UtcNow; } }
private static void DumpStackTraces(ZipArchive archive, string prefix) { var zipArchiveEntry = archive.CreateEntry($"{prefix}/stacktraces.json", CompressionLevel.Optimal); var threadsUsage = new ThreadsUsage(); var sp = Stopwatch.StartNew(); using (var stackTraceStream = zipArchiveEntry.Open()) using (var sw = new StringWriter()) { try { if (Debugger.IsAttached) { throw new InvalidOperationException("Cannot get stack traces when debugger is attached"); } ThreadsHandler.OutputResultToStream(sw); var result = JObject.Parse(sw.GetStringBuilder().ToString()); var wait = 100 - sp.ElapsedMilliseconds; if (wait > 0) { // I expect this to be _rare_, but we need to wait to get a correct measure of the cpu Thread.Sleep((int)wait); } var threadStats = threadsUsage.Calculate(); result["Threads"] = JArray.FromObject(threadStats.List); using (var writer = new StreamWriter(stackTraceStream)) { result.WriteTo(new JsonTextWriter(writer) { Indentation = 4 }); writer.Flush(); } } catch (Exception e) { var jsonSerializer = DocumentConventions.Default.CreateSerializer(); jsonSerializer.Formatting = Formatting.Indented; using (var errorSw = new StreamWriter(stackTraceStream)) { jsonSerializer.Serialize(errorSw, new { Error = e.Message }); } } } }
public Task StackTrace() { if (PlatformDetails.RunningOnMacOsx) { throw new NotSupportedException("Capturing live stack traces is not supported by RavenDB on MacOSX"); } var threadIds = GetStringValuesQueryString("threadId", required: false); var includeStackObjects = GetBoolValueQueryString("includeStackObjects", required: false) ?? false; var sp = Stopwatch.StartNew(); var threadsUsage = new ThreadsUsage(); using (var sw = new StringWriter()) { OutputResultToStream(sw, threadIds.ToHashSet(), includeStackObjects); var result = JObject.Parse(sw.GetStringBuilder().ToString()); var wait = 100 - sp.ElapsedMilliseconds; if (wait > 0) { // I expect this to be _rare_, but we need to wait to get a correct measure of the cpu Thread.Sleep((int)wait); } var threadStats = threadsUsage.Calculate(); result["Threads"] = JArray.FromObject(threadStats.List); using (var writer = new StreamWriter(ResponseBodyStream())) { result.WriteTo(new JsonTextWriter(writer) { Indentation = 4 }); writer.Flush(); } } return(Task.CompletedTask); }
public void ThreadUsage_WhenThreadsHaveSameCpuUsageAndTotalProcessorTime_ShouldListThemBoth() { using var database = CreateDocumentDatabase(); using var index1 = MapIndex.CreateNew(new IndexDefinition { Name = "Companies_ByName", Maps = { "from company in docs.Companies select new { company.Name }" }, }, database); using var index2 = MapIndex.CreateNew(new IndexDefinition { Name = "Users_ByName", Maps = { "from user in docs.Orders select new { user.Name }" }, }, database); using var index3 = MapIndex.CreateNew(new IndexDefinition { Name = "Orders_ByName", Maps = { "from order in docs.Orders select new { order.Name }" }, }, database); index1.Start(); index2.Start(); index3.Start(); for (int i = 0;; i++) { try { var threadsUsage = new ThreadsUsage(); var threadsInfo = threadsUsage.Calculate(); var threadNames = threadsInfo.List.Select(ti => ti.Name).OrderBy(n => n).ToArray(); RavenTestHelper.AssertAll(() => string.Join('\n', threadNames.Select(s => $"\"{s}\"")), () => AssertContains(index1._indexingThread.Name), () => AssertContains(index2._indexingThread.Name), () => AssertContains(index3._indexingThread.Name)); break; void AssertContains(string threadName) => Assert.True(threadNames.Contains(threadName), $"Not found : {threadName}"); } catch { if (i >= 5) { throw; } Thread.Sleep(100); } } }
public async Task StackTrace() { if (PlatformDetails.RunningOnMacOsx) { throw new NotSupportedException("Capturing live stack traces is not supported by RavenDB on MacOSX"); } var threadIds = GetStringValuesQueryString("threadId", required: false); var includeStackObjects = GetBoolValueQueryString("includeStackObjects", required: false) ?? false; var sp = Stopwatch.StartNew(); var threadsUsage = new ThreadsUsage(); using (var sw = new StringWriter()) { OutputResultToStream(sw, threadIds.ToHashSet(), includeStackObjects); var result = JObject.Parse(sw.GetStringBuilder().ToString()); var wait = 100 - sp.ElapsedMilliseconds; if (wait > 0) { // I expect this to be _rare_, but we need to wait to get a correct measure of the cpu await Task.Delay((int)wait); } var threadStats = threadsUsage.Calculate(); result["Threads"] = JArray.FromObject(threadStats.List); using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, DocumentConventions.DefaultForServer.Serialization.DefaultConverter.ToBlittable(result, context)); } } } }
public static void WriteThreadsInfoAndWaitForEsc(RavenServer server, int maxTopThreads, int updateIntervalInMs, double cpuUsageThreshold) { Console.WriteLine("Showing threads info, press any key to close..."); var i = 0L; var threadsUsage = new ThreadsUsage(); Thread.Sleep(100); var cursorLeft = Console.CursorLeft; var cursorTop = Console.CursorTop; var waitIntervals = updateIntervalInMs / 100; var maxNameLength = 0; while (Console.KeyAvailable == false) { Console.SetCursorPosition(cursorLeft, cursorTop); var threadsInfo = threadsUsage.Calculate(); Console.Write($"{(i++ % 2 == 0 ? "*" : "+")} "); Console.WriteLine($"CPU usage: {threadsInfo.CpuUsage:0.00}% (total threads: {threadsInfo.List.Count:#,#0}, active cores: {threadsInfo.ActiveCores}) "); var printedLines = 1; var count = 0; var isFirst = true; foreach (var threadInfo in threadsInfo.List .Where(x => x.CpuUsage >= cpuUsageThreshold)) { if (isFirst) { printedLines++; Console.WriteLine(" thread id | cpu usage | priority | thread name "); isFirst = false; } if (++count > maxTopThreads) { break; } var nameLength = threadInfo.Name.Length; maxNameLength = Math.Max(maxNameLength, nameLength); var numberOfEmptySpaces = maxNameLength - nameLength + 1; var emptySpaces = numberOfEmptySpaces > 0 ? new string(' ', numberOfEmptySpaces) : string.Empty; Console.Write($" {threadInfo.Id,-7} "); Console.Write($" | {$"{threadInfo.CpuUsage:0.00}%",-8}"); Console.Write($" | {threadInfo.Priority,-12} "); Console.Write($" | {threadInfo.Name}{emptySpaces}"); Console.WriteLine(); printedLines++; } for (var j = 0; j < waitIntervals && Console.KeyAvailable == false; j++) { Thread.Sleep(100); } if (PlatformDetails.RunningOnPosix) { var newTop = Console.BufferHeight - cursorTop - printedLines - 1; if (newTop < 0) { cursorTop = Math.Max(0, cursorTop + newTop); } } } Console.ReadKey(true); Console.WriteLine(); Console.WriteLine("Threads info halted."); }