public DocumentCacher(InMemoryRavenConfiguration configuration)
{
this.configuration = configuration;
cachedSerializedDocuments = CreateCache();
MemoryStatistics.RegisterLowMemoryHandler(this);
}
public IndexSearcherHoldingState(IndexSearcher indexSearcher, string publicName, string databaseName)
{
IndexSearcher = indexSearcher;
this.databaseName = databaseName;
indexName = publicName;
MemoryStatistics.RegisterLowMemoryHandler(this);
}
public static Measurement Measure(string name, [NotNull] Action action, int blocks)
{
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var cpuUserTimeBefore = CurrentProcess.UserProcessorTime;
var cpuKernelTimeBefore = CurrentProcess.PrivilegedProcessorTime;
var memoryBefore = MemoryStatistics.TakeCurrent();
var stopwatch = Stopwatch.StartNew();
action();
stopwatch.Stop();
var memoryAfter = MemoryStatistics.TakeCurrent();
var cpuUserTimeAfter = CurrentProcess.UserProcessorTime;
var cpuKernelTimeAfter = CurrentProcess.PrivilegedProcessorTime;
var gcDelta = MemoryStatistics.Delta(memoryBefore, memoryAfter);
return(new Measurement(
name,
blocks,
stopwatch.Elapsed,
(cpuKernelTimeAfter - cpuKernelTimeBefore),
(cpuUserTimeAfter - cpuUserTimeBefore),
gcDelta));
}
public BenchmarkTest(Database database, int flowCount, long recordCount, float randomness, CancellationTokenSource cancellation)
{
Database = database;
FlowCount = flowCount;
RecordCount = recordCount;
Randomness = randomness;
KeysType = Randomness == 0f ? KeysType.Sequential : KeysType.Random;
// Statistics.
int length = Enum.GetValues(typeof(TestMethod)).Length - 1;
SpeedStatistics = new SpeedStatistics[length];
MemoryStatistics = new MemoryStatistics[length];
int step = (int)((recordCount) / INTERVAL_COUNT);
for (int i = 0; i < length; i++)
{
SpeedStatistics[i] = new SpeedStatistics(INTERVAL_COUNT);
SpeedStatistics[i].Step = step;
MemoryStatistics[i] = new MemoryStatistics(INTERVAL_COUNT);
MemoryStatistics[i].Step = step;
}
Cancellation = cancellation;
}
public PerformanceWatch(string name, int step)
{
this.name = name;
SpeedStatistics = new SpeedStatistics(Benchmark.INTERVAL_COUNT, step);
MemoryStatistics = new MemoryStatistics(Benchmark.INTERVAL_COUNT, step);
}
public MacFaceApp()
{
config = Configuration.GetInstance();
config.Load();
pageio_count = 0;
cpuStats = new CPUStatisticsNtQuerySystemInformation(61);
memStats = new MemoryStatisticsNtQuerySystemInformation(61);
patternWindow = null;
statusWindow = null;
InitializeComponent();
// x64 環境で 32bit な OptimusMini.dll を読み込もうとすると当然エラーとなるので何もしないクラスにしておく
optimusMini = (IntPtr.Size == 4 ? (IOptimusMini) new OptimusMini() : new OptimusMiniMock());
optimusMini.DisplayOn();
//OptimusMini.OnKeyDownCallbackDelegate oKD = new OptimusMini.OnKeyDownCallbackDelegate(OnKeyDownCallbackHandler);
//OptimusMini.OnDeviceStateChangedCallbackDelegate oDSC = new OptimusMini.OnDeviceStateChangedCallbackDelegate(OnDeviceStateChangedCallbackHandler);
//OptimusMini.RegisterEventHandler(oKD, oDSC);
CountProcessorUsage(null, null);
updateTimer = new System.Windows.Forms.Timer();
updateTimer.Enabled = false;
updateTimer.Interval = 1000;
updateTimer.Tick += new EventHandler(this.CountProcessorUsage);
}
public PrefetchingBehavior(PrefetchingUser prefetchingUser, WorkContext context, BaseBatchSizeAutoTuner autoTuner)
{
this.context = context;
this.autoTuner = autoTuner;
PrefetchingUser = prefetchingUser;
MemoryStatistics.RegisterLowMemoryHandler(this);
}
protected BaseBatchSizeAutoTuner(WorkContext context)
{
this.context = context;
NumberOfItemsToIndexInSingleBatch = InitialNumberOfItems;
MemoryStatistics.RegisterLowMemoryHandler(this);
_currentlyUsedBatchSizesInBytes = new ConcurrentDictionary <Guid, long>();
memoryLimitForIndexingInBytes = context.Configuration.MemoryLimitForIndexingInMB * 1024 * 1024;
}
private Measurement(string name, int blocks, TimeSpan elapsed, TimeSpan kernelTime, TimeSpan userTime, MemoryStatistics gcDelta)
{
_name = name;
_blocks = blocks;
_elapsed = elapsed;
_kernelTime = kernelTime;
_userTime = userTime;
_gcDelta = gcDelta;
}
public HttpResponseMessage LowMemoryNotification()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Low memory simulation is only possible from the system database", HttpStatusCode.BadRequest));
}
MemoryStatistics.SimulateLowMemoryNotification();
return(GetEmptyMessage());
}
protected BaseBatchSizeAutoTuner(WorkContext context)
{
this.context = context;
FetchingDocumentsFromDiskTimeout = TimeSpan.FromSeconds(context.Configuration.Prefetcher.FetchingDocumentsFromDiskTimeoutInSeconds);
maximumSizeAllowedToFetchFromStorageInMb = context.Configuration.Prefetcher.MaximumSizeAllowedToFetchFromStorageInMb;
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
NumberOfItemsToProcessInSingleBatch = InitialNumberOfItems;
MemoryStatistics.RegisterLowMemoryHandler(this);
context.TransactionalStorage.RegisterTransactionalStorageNotificationHandler(this);
_currentlyUsedBatchSizesInBytes = new ConcurrentDictionary <Guid, long>();
}
public static MemoryStatistics Delta([NotNull] MemoryStatistics before, [NotNull] MemoryStatistics after)
{
if (before == null)
{
throw new ArgumentNullException(nameof(before));
}
if (after == null)
{
throw new ArgumentNullException(nameof(after));
}
return(new MemoryStatistics(after.Gen0 - before.Gen0, after.Gen1 - before.Gen1, after.Gen2 - before.Gen2));
}
public void get_debug_info_ForSpecifiedDatabaseReducedBatchSizeAutoTuner()
{
using (var documentStore = NewDocumentStore())
{
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "Dba1",
Settings =
{
{ "Raven/DataDir", "Dba1" }
}
});
var dbWorkContext = documentStore.ServerIfEmbedded.Options.DatabaseLandlord.GetResourceInternal("Dba1").Result.WorkContext;
var dbName = dbWorkContext.DatabaseName;
var reduceBatchSizeAutoTuner = new ReduceBatchSizeAutoTuner(dbWorkContext);
var prefetchingBehavior = new PrefetchingBehavior(PrefetchingUser.Indexer, dbWorkContext, reduceBatchSizeAutoTuner, string.Empty);
MemoryStatistics.SimulateLowMemoryNotification();
reduceBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue);
reduceBatchSizeAutoTuner.HandleLowMemory();
prefetchingBehavior.HandleLowMemory();
MemoryStatistics.RunLowMemoryHandlers("System detected low memory");
prefetchingBehavior.OutOfMemoryExceptionHappened();
prefetchingBehavior.HandleLowMemory();
var url = $"http://localhost:8079/databases/{dbName}/debug/auto-tuning-info";
var requestWithDbName = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get,
documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions));
var results = requestWithDbName.ReadResponseJson().JsonDeserialization <AutoTunerInfo>();
var reason = results.Reason;
var reasonForLowMemoryCall = results.LowMemoryCallsRecords.First().Reason;
var lowMemoryRecords = results.LowMemoryCallsRecords.First().Operations;
Assert.Equal("System detected low memory", reasonForLowMemoryCall);
var urlAdmin = $"http://localhost:8079/databases/{dbName}/admin/debug/auto-tuning-info";
var requestWithDbNameAdmin = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, urlAdmin, HttpMethods.Get,
documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions));
var resultsAdmin = requestWithDbNameAdmin.ReadResponseJson().JsonDeserialization <AutoTunerInfo>();
var reasonAdmin = results.Reason;
var reasonForLowMemoryCallAdmin = resultsAdmin.LowMemoryCallsRecords.First().Reason;
var lowMemoryRecordsAdmin = resultsAdmin.LowMemoryCallsRecords.First().Operations;
var cpuUsageRecordsAdmin = resultsAdmin.CpuUsageCallsRecords;
Assert.Equal("System detected low memory", reasonForLowMemoryCallAdmin);
}
}
public StatusWindow(CPUStatistics cpuStats, MemoryStatistics memStats)
{
//
// Windows フォーム デザイナ サポートに必要です。
//
InitializeComponent();
this.cpuStats = cpuStats;
this.memStats = memStats;
cpuGraph = new Bitmap(cpuGraphPicBox.Width, cpuGraphPicBox.Height);
memoryGraph = new Bitmap(memoryGraphPicBox.Width, memoryGraphPicBox.Height);
cpuGraphPicBox.Image = cpuGraph;
memoryGraphPicBox.Image = memoryGraph;
}
public void get_debug_info_ForAdmin()
{
using (var documentStore = NewDocumentStore())
{
var workContext = documentStore.SystemDatabase.WorkContext;
var prefetchingBehavior = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new IndexBatchSizeAutoTuner(workContext), string.Empty);
MemoryStatistics.RunLowMemoryHandlers("System detected low memory");
var prefetchingBehavior2 = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new IndependentBatchSizeAutoTuner(workContext, PrefetchingUser.Indexer), string.Empty);
var prefetchingBehavior3 = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new ReduceBatchSizeAutoTuner(workContext), string.Empty);
var indexBatchAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior.PrefetchingUser);
var independentBatchSizeAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior2.PrefetchingUser);
var reduceBatchSizeAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior3.PrefetchingUser);
independentBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue);
reduceBatchSizeAutoTuner.AutoThrottleBatchSize(500, 1024, TimeSpan.MinValue);
indexBatchAutoTuner.HandleLowMemory();
reduceBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue);
prefetchingBehavior3.HandleLowMemory();
indexBatchAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue);
prefetchingBehavior2.HandleLowMemory();
prefetchingBehavior.OutOfMemoryExceptionHappened();
prefetchingBehavior.HandleLowMemory();
MemoryStatistics.RunLowMemoryHandlers("System detected low memory");
var url = "http://localhost:8079/admin/debug/auto-tuning-info";
var requestWithDbName = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get,
documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions));
var results = requestWithDbName.ReadResponseJson().JsonDeserialization <AutoTunerInfo>();
var reason = results.Reason;
var reasonForLowMemoryCall = results.LowMemoryCallsRecords.First().Reason;
var lowMemoryRecords = results.LowMemoryCallsRecords.First().Operations;
Assert.Equal("System detected low memory", reasonForLowMemoryCall);
}
}
private static void ReleaseMemoryBeforeGC()
{
if (MemoryStatistics.AvailableMemory < (MemoryStatistics.TotalPhysicalMemory - MemoryStatistics.AvailableMemory) / 10)
{
if (Environment.TickCount - lastTimeMemoryReleasedBeforeGC < fiveSecondsInTicks)
{
return;
}
lastTimeMemoryReleasedBeforeGC = Environment.TickCount;
MemoryStatistics.SimulateLowMemoryNotification();
}
else
{
MemoryStatistics.InitiateSoftMemoryRelease();
}
}
public MemoryStatistics GetMemoryStatistics()
{
var result = new MemoryStatistics {
UnmanagedVertexBytes = TotalVertexBytes,
UnmanagedIndexBytes = TotalIndexBytes,
ManagedVertexBytes = 0,
ManagedIndexBytes = 0
};
lock (_AllBufferGenerators)
foreach (var generator in _AllBufferGenerators)
{
result.ManagedVertexBytes += generator.ManagedVertexBytes;
result.ManagedIndexBytes += generator.ManagedIndexBytes;
}
return(result);
}
public HttpResponseMessage GetLowMemoryStatistics()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Low memory simulation is only possible from the system database", HttpStatusCode.BadRequest));
}
return(GetMessageWithObject(MemoryStatistics.GetLowMemoryHandlersStatistics().GroupBy(x => x.DatabaseName).Select(x => new
{
DatabaseName = x.Key,
Types = x.GroupBy(y => y.Name).Select(y => new
{
MemoryHandlerName = y.Key,
MemoryHandlers = y.Select(z => new {
z.EstimatedUsedMemory,
z.Metadata
})
})
})));
}
public CachedIndexedTerms()
{
MemoryStatistics.RegisterLowMemoryHandler(this);
}
public void get_debug_info_ForSpecifiedDatabase()
{
using (var documentStore = NewDocumentStore())
{
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "Dba1",
Settings =
{
{ "Raven/DataDir", "Dba1" }
}
});
new Simple().Execute(documentStore.DatabaseCommands.ForDatabase("Dba1"), documentStore.Conventions);
using (var session = documentStore.OpenSession("Dba1"))
{
var entity1 = new Person {
Name = "Rob", Email = "*****@*****.**"
};
var entity2 = new Person {
Name = "Haim", Email = "*****@*****.**"
};
var entity3 = new Person {
Name = "Rob", Email = "*****@*****.**"
};
session.Store(entity1);
session.Store(entity2);
session.Store(entity3);
var persons = session.Query <Person>().Where(x => x.Name == "Rob").ToList();
}
var dbWorkContext = documentStore.ServerIfEmbedded.Options.DatabaseLandlord.GetResourceInternal("Dba1").Result.WorkContext;
var dbName = dbWorkContext.DatabaseName;
var independentBatchSizeAutoTuner = new IndexBatchSizeAutoTuner(dbWorkContext);
var prefetchingBehavior = new PrefetchingBehavior(PrefetchingUser.Indexer, dbWorkContext, independentBatchSizeAutoTuner, string.Empty);
prefetchingBehavior.HandleLowMemory();
independentBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue);
independentBatchSizeAutoTuner.HandleLowMemory();
MemoryStatistics.RunLowMemoryHandlers("System detected low memory");
var url = $"http://localhost:8079/databases/{dbName}/debug/auto-tuning-info";
var requestWithDbName = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get,
documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions));
var results = requestWithDbName.ReadResponseJson().JsonDeserialization <AutoTunerInfo>();
var reason = results.Reason;
var reasonForLowMemoryCall = results.LowMemoryCallsRecords.First().Reason;
var lowMemoryRecords = results.LowMemoryCallsRecords.First().Operations;
Assert.Equal("System detected low memory", reasonForLowMemoryCall);
var urlAdmin = $"http://localhost:8079/databases/{dbName}/admin/debug/auto-tuning-info";
var requestWithDbNameAdmin = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get,
documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions));
var resultsAdmin = requestWithDbNameAdmin.ReadResponseJson().JsonDeserialization <AutoTunerInfo>();
var reasonAdmin = results.Reason;
var reasonForLowMemoryCallAdmin = resultsAdmin.LowMemoryCallsRecords.First().Reason;
var lowMemoryRecordsAdmin = resultsAdmin.LowMemoryCallsRecords.First().Operations;
var cpuUsageRecordsAdmin = resultsAdmin.CpuUsageCallsRecords;
Assert.Equal("System detected low memory", reasonForLowMemoryCallAdmin);
}
}
static DocumentIdSet()
{
MemoryStatistics.RegisterLowMemoryHandler(documentIdSetCleaner);
}
static WebSocketTransportFactory()
{
MemoryStatistics.RegisterLowMemoryHandler(DisconnectWebSockets.Instance);
}
public static MemoryStatistics Statistics() => MemoryStatistics.Get();
private void ExecutedMemoryStatistics(object sender, ExecutedRoutedEventArgs e)
{
MemoryStatistics.AddOrRemove();
IsMemoryStatEnabled = MemoryStatistics.IsEnabled;
}
public IndexSearcherHoldingState(IndexSearcher indexSearcher)
{
IndexSearcher = indexSearcher;
MemoryStatistics.RegisterLowMemoryHandler(this);
}
private void ExecutedMemoryStatisticsCommand(object sender, ExecutedRoutedEventArgs e)
{
MemoryStatistics.AddOrRemove();
MemoryStatMenuItem.IsChecked = MemoryStatistics.IsEnabled;
}
public Prefetcher(WorkContext workContext)
{
this.workContext = workContext;
MemoryStatistics.RegisterLowMemoryHandler(this);
}
protected IEnumerator<object> CaptureSnapshotTask(string targetFilename)
{
var now = DateTime.Now;
var mem = new MemoryStatistics(Process);
var psi = new ProcessStartInfo(
Settings.UmdhPath, String.Format(
"-g -p:{0} -f:\"{1}\"", Process.Id, targetFilename
)
);
IDictionary<string, string> environment = null;
yield return GetEnvironment().Bind(() => environment);
TemporaryFiles.Add(targetFilename);
using (Activities.AddItem("Capturing heap snapshot"))
yield return Program.RunProcess(psi, customEnvironment: environment);
yield return Future.RunInThread(
() => File.AppendAllText(targetFilename, mem.GetFileText())
);
var fText = Future.RunInThread(
() => File.ReadAllText(targetFilename)
);
yield return fText;
var text = fText.Result;
fText = null;
yield return FinishLoadingSnapshot(Snapshots.Count, now, targetFilename, text);
}
public CachedIndexedTerms(string databaseName, string indexName)
{
this.databaseName = databaseName;
this.indexName = indexName;
MemoryStatistics.RegisterLowMemoryHandler(this);
}
public WeakCache()
{
MemoryStatistics.RegisterLowMemoryHandler(this);
}
