public async Task RunawayThreads()
{
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
try
{
var threadsUsage = new ThreadsUsage();
// need to wait to get a correct measure of the cpu
await Task.Delay(100);
var result = threadsUsage.Calculate();
context.Write(writer,
new DynamicJsonValue
{
["Runaway Threads"] = result.ToJson()
});
}
catch (Exception e)
{
context.Write(writer,
new DynamicJsonValue
{
["Error"] = e.ToString()
});
}
writer.Flush();
}
}
}
public ThreadsInfoNotificationSender(string resourceName,
ConcurrentSet <ConnectedWatcher> watchers, TimeSpan notificationsThrottle, CancellationToken shutdown)
: base(resourceName, shutdown)
{
_watchers = watchers;
_notificationsThrottle = notificationsThrottle;
_threadsUsage = new ThreadsUsage();
}
private static void DumpStackTraces(ZipArchive archive, string prefix)
{
var zipArchiveEntry = archive.CreateEntry($"{prefix}/stacktraces.json", CompressionLevel.Optimal);
var threadsUsage = new ThreadsUsage();
var sp = Stopwatch.StartNew();
using (var stackTraceStream = zipArchiveEntry.Open())
using (var sw = new StringWriter())
{
try
{
if (Debugger.IsAttached)
{
throw new InvalidOperationException("Cannot get stack traces when debugger is attached");
}
ThreadsHandler.OutputResultToStream(sw);
var result = JObject.Parse(sw.GetStringBuilder().ToString());
var wait = 100 - sp.ElapsedMilliseconds;
if (wait > 0)
{
// I expect this to be _rare_, but we need to wait to get a correct measure of the cpu
Thread.Sleep((int)wait);
}
var threadStats = threadsUsage.Calculate();
result["Threads"] = JArray.FromObject(threadStats.List);
using (var writer = new StreamWriter(stackTraceStream))
{
result.WriteTo(new JsonTextWriter(writer)
{
Indentation = 4
});
writer.Flush();
}
}
catch (Exception e)
{
var jsonSerializer = DocumentConventions.Default.CreateSerializer();
jsonSerializer.Formatting = Formatting.Indented;
using (var errorSw = new StreamWriter(stackTraceStream))
{
jsonSerializer.Serialize(errorSw, new
{
Error = e.Message
});
}
}
}
}
public Task StackTrace()
{
if (PlatformDetails.RunningOnMacOsx)
{
throw new NotSupportedException("Capturing live stack traces is not supported by RavenDB on MacOSX");
}
var threadIds = GetStringValuesQueryString("threadId", required: false);
var includeStackObjects = GetBoolValueQueryString("includeStackObjects", required: false) ?? false;
var sp = Stopwatch.StartNew();
var threadsUsage = new ThreadsUsage();
using (var sw = new StringWriter())
{
OutputResultToStream(sw, threadIds.ToHashSet(), includeStackObjects);
var result = JObject.Parse(sw.GetStringBuilder().ToString());
var wait = 100 - sp.ElapsedMilliseconds;
if (wait > 0)
{
// I expect this to be _rare_, but we need to wait to get a correct measure of the cpu
Thread.Sleep((int)wait);
}
var threadStats = threadsUsage.Calculate();
result["Threads"] = JArray.FromObject(threadStats.List);
using (var writer = new StreamWriter(ResponseBodyStream()))
{
result.WriteTo(new JsonTextWriter(writer)
{
Indentation = 4
});
writer.Flush();
}
}
return(Task.CompletedTask);
}
public void ThreadUsage_WhenThreadsHaveSameCpuUsageAndTotalProcessorTime_ShouldListThemBoth()
{
using var database = CreateDocumentDatabase();
using var index1 = MapIndex.CreateNew(new IndexDefinition { Name = "Companies_ByName", Maps = { "from company in docs.Companies select new { company.Name }" }, }, database);
using var index2 = MapIndex.CreateNew(new IndexDefinition { Name = "Users_ByName", Maps = { "from user in docs.Orders select new { user.Name }" }, }, database);
using var index3 = MapIndex.CreateNew(new IndexDefinition { Name = "Orders_ByName", Maps = { "from order in docs.Orders select new { order.Name }" }, }, database);
index1.Start();
index2.Start();
index3.Start();
for (int i = 0;; i++)
{
try
{
var threadsUsage = new ThreadsUsage();
var threadsInfo = threadsUsage.Calculate();
var threadNames = threadsInfo.List.Select(ti => ti.Name).OrderBy(n => n).ToArray();
RavenTestHelper.AssertAll(() => string.Join('\n', threadNames.Select(s => $"\"{s}\"")),
() => AssertContains(index1._indexingThread.Name),
() => AssertContains(index2._indexingThread.Name),
() => AssertContains(index3._indexingThread.Name));
break;
void AssertContains(string threadName) => Assert.True(threadNames.Contains(threadName), $"Not found : {threadName}");
}
catch
{
if (i >= 5)
{
throw;
}
Thread.Sleep(100);
}
}
}
public async Task StackTrace()
{
if (PlatformDetails.RunningOnMacOsx)
{
throw new NotSupportedException("Capturing live stack traces is not supported by RavenDB on MacOSX");
}
var threadIds = GetStringValuesQueryString("threadId", required: false);
var includeStackObjects = GetBoolValueQueryString("includeStackObjects", required: false) ?? false;
var sp = Stopwatch.StartNew();
var threadsUsage = new ThreadsUsage();
using (var sw = new StringWriter())
{
OutputResultToStream(sw, threadIds.ToHashSet(), includeStackObjects);
var result = JObject.Parse(sw.GetStringBuilder().ToString());
var wait = 100 - sp.ElapsedMilliseconds;
if (wait > 0)
{
// I expect this to be _rare_, but we need to wait to get a correct measure of the cpu
await Task.Delay((int)wait);
}
var threadStats = threadsUsage.Calculate();
result["Threads"] = JArray.FromObject(threadStats.List);
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, DocumentConventions.DefaultForServer.Serialization.DefaultConverter.ToBlittable(result, context));
}
}
}
}
public static void WriteThreadsInfoAndWaitForEsc(RavenServer server, int maxTopThreads, int updateIntervalInMs, double cpuUsageThreshold)
{
Console.WriteLine("Showing threads info, press any key to close...");
var i = 0L;
var threadsUsage = new ThreadsUsage();
Thread.Sleep(100);
var cursorLeft = Console.CursorLeft;
var cursorTop = Console.CursorTop;
var waitIntervals = updateIntervalInMs / 100;
var maxNameLength = 0;
while (Console.KeyAvailable == false)
{
Console.SetCursorPosition(cursorLeft, cursorTop);
var threadsInfo = threadsUsage.Calculate();
Console.Write($"{(i++ % 2 == 0 ? "*" : "+")} ");
Console.WriteLine($"CPU usage: {threadsInfo.CpuUsage:0.00}% (total threads: {threadsInfo.List.Count:#,#0}, active cores: {threadsInfo.ActiveCores}) ");
var printedLines = 1;
var count = 0;
var isFirst = true;
foreach (var threadInfo in threadsInfo.List
.Where(x => x.CpuUsage >= cpuUsageThreshold))
{
if (isFirst)
{
printedLines++;
Console.WriteLine(" thread id | cpu usage | priority | thread name ");
isFirst = false;
}
if (++count > maxTopThreads)
{
break;
}
var nameLength = threadInfo.Name.Length;
maxNameLength = Math.Max(maxNameLength, nameLength);
var numberOfEmptySpaces = maxNameLength - nameLength + 1;
var emptySpaces = numberOfEmptySpaces > 0 ? new string(' ', numberOfEmptySpaces) : string.Empty;
Console.Write($" {threadInfo.Id,-7} ");
Console.Write($" | {$"{threadInfo.CpuUsage:0.00}%",-8}");
Console.Write($" | {threadInfo.Priority,-12} ");
Console.Write($" | {threadInfo.Name}{emptySpaces}");
Console.WriteLine();
printedLines++;
}
for (var j = 0; j < waitIntervals && Console.KeyAvailable == false; j++)
{
Thread.Sleep(100);
}
if (PlatformDetails.RunningOnPosix)
{
var newTop = Console.BufferHeight - cursorTop - printedLines - 1;
if (newTop < 0)
{
cursorTop = Math.Max(0, cursorTop + newTop);
}
}
}
Console.ReadKey(true);
Console.WriteLine();
Console.WriteLine("Threads info halted.");
}
