private static void MainPersons(string[] args)
{
TestingPhotoPersons.Npersons = 40 * 1000;
string path = "../../../Databases/int based/" + TestingPhotoPersons.Npersons / 1000 + "/";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
StoreCascadingInt store = new StoreCascadingInt(path);
using (StreamWriter perfomance = new StreamWriter("../../Perfomance.txt"))
perfomance.WriteLine(TestingPhotoPersons.Npersons);
if (true)
{
Performance.ComputeTime(() => Reload(store), "load " + TestingPhotoPersons.Npersons + " ", true);
}
TestingPhotoPersons.Run((q) =>
{
var sparqlQuery = SparqlQueryParser.Parse(store, q).Run();
if (sparqlQuery.ResultType == ResultType.Ask)
{
var b = sparqlQuery.AnyResult;
}
else
{
sparqlQuery.Results.Count();
}
});
}
public static void RunBerlinsWithConstants()
{
double[] memoryUsage = new double[12];
double[] totalparseMS = new double[12];
double[] totalCreateLinqStack = new double[12];
double[] totalrun = new double[12];
Console.WriteLine("bsbm with constants");
var timer = new Stopwatch();
// using (
var Store = new Store("../../../Databases/int based/");
Store.Start();
//Store.ActivateCache();
{
// Store.ReloadFrom(Config.Source_data_folder_path+"1.ttl");
SparqlQueryParser.Parse(Store, sq5);
for (
int i = 0;
i < 12; i++)
{
string file = string.Format(@"..\..\examples\bsbm\queries\with constants\{0}.rq", i + 1);
var readAllText = File.ReadAllText(file);
GC.Collect();
timer.Restart();
var sparqlQuery = SparqlQueryParser.Parse(Store, readAllText);
timer.Stop();
totalparseMS[i] += GetTimeWthLast2Digits(timer);
timer.Restart();
var sparqlResultSet = sparqlQuery.Run();
timer.Stop();
totalCreateLinqStack[i] += GetTimeWthLast2Digits(timer);
timer.Restart();
sparqlResultSet.Results.ToArray();
timer.Stop();
totalrun[i] += GetTimeWthLast2Digits(timer);
memoryUsage[i] = GC.GetTotalMemory(false);
File.WriteAllText(Path.ChangeExtension(file, ".json"), sparqlResultSet.ToJson());
//results[i]=
}
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine("date time " + DateTime.Now);
r.WriteLine("memory usage (bytes)" + string.Join(", ", memoryUsage));
r.WriteLine("parse (ms)" + string.Join(", ", totalparseMS));
r.WriteLine("total parse " + totalparseMS.Sum());
r.WriteLine("create linq stack " + string.Join(", ", totalCreateLinqStack));
r.WriteLine("total create linq stack " + totalCreateLinqStack.Sum());
r.WriteLine("run " + string.Join(", ", totalrun));
r.WriteLine("parse + create linq stack + run " + string.Join(", ", totalrun.Select((d, i) => d + totalCreateLinqStack[i] + totalparseMS[i]).Select(d => d.ToString().Replace(",", "."))));
r.WriteLine("total run " + totalrun.Sum());
r.WriteLine("total " +
(totalparseMS.Sum() + totalCreateLinqStack.Sum() + totalrun.Sum()));
}
}
public static void RunTestParametred(int count = 100)
{
var Store = new Store("../../../Databases/int based/");
// Store.ClearAll();
// Store.ReloadFrom(10*1000*1000, Config.Source_data_folder_path + "10M.ttl");
// SparqlQueryParser.Parse(Store, sq5);
Store.Start();
Store.ActivateCache();
for (int i = 0; i < 12; i++)
{
BSBmParams bsBmParams = new BSBmParams(Store);
var paramvaluesFilePath =
string.Format(@"..\..\examples\bsbm\queries\parameters\param values for{0}m {1} query.txt", 1, i + 1);
var qFile =
string.Format(@"..\..\examples\bsbm\queries\parameters\{0}.rq", i + 1);
using (StreamReader streamParameters = new StreamReader(paramvaluesFilePath))
using (StreamReader streamQuery = new StreamReader(qFile))
{
string qparams = streamQuery.ReadToEnd();
Stopwatch timer = new Stopwatch();
for (int j = 0; j < count; j++)
{
string q = BSBmParams.QueryReadParameters(qparams, streamParameters);
var sparqlResults = SparqlQueryParser.Parse(Store, q).Run();
timer.Start();
sparqlResults.Results.Count();
timer.Stop();
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine();
r.WriteLine("one query {0}, {1} times", i + 1, count);
r.WriteLine("milions " + 1);
r.WriteLine("date time " + DateTime.Now);
r.WriteLine("total ms " + timer.ElapsedMilliseconds);
double l = ((double)timer.ElapsedMilliseconds) / count;
r.WriteLine("ms " + l);
r.WriteLine("qps " + (int)(1000.0 / l));
string q = BSBmParams.QueryReadParameters(qparams, streamParameters);
r.WriteLine("next results count: {0}",
SparqlQueryParser.Parse(Store, q).Run().Results.Count());
}
}
}
}
public void Examples()
{
DirectoryInfo examplesRoot = new DirectoryInfo(@"..\..\examples");
foreach (var exampleDir in examplesRoot.GetDirectories().Skip(0))
// var exampleDir = new DirectoryInfo(@"..\..\examples\bsbm");
{
Console.WriteLine("example: " + exampleDir.Name);
//if (exampleDir.Name != @"federated subquery"
// //&& rqQueryFile.FullName != @"C:\Users\Admin\Source\Repos\SparqlWpf\UnitTestDotnetrdf_test\examples\insert where\query2.rq"
// ) continue;
//var nameGraphsDir = new DirectoryInfo(Path.Combine(exampleDir.FullName, "named graphs"));
//if (nameGraphsDir.Exists) continue;
foreach (var ttlDatabase in exampleDir.GetFiles("*.ttl"))
{
var store = new Store(exampleDir.FullName + "/tmp");
store.ClearAll();
//using (StreamReader reader = new StreamReader(ttlDatabase.FullName))
store.ReloadFrom(ttlDatabase.FullName);
// store.Start();
var nameGraphsDir = new DirectoryInfo(Path.Combine(exampleDir.FullName, "named graphs"));
if (nameGraphsDir.Exists)
{
foreach (var namedGraphFile in nameGraphsDir.GetFiles())
{
IGraph graph;
using (StreamReader reader = new StreamReader(namedGraphFile.FullName))
{
var readLine = reader.ReadLine();
if (readLine == null)
{
continue;
}
var headComment = readLine.Trim();
if (!headComment.StartsWith("#"))
{
continue;
}
headComment = headComment.Substring(1);
//Uri uri;
//if (!Uri.TryCreate(headComment, UriKind.Absolute, out uri)) continue;Prologue.SplitUri(uri.AbsoluteUri).FullName
graph = store.NamedGraphs.CreateGraph(headComment);
}
graph.FromTurtle(namedGraphFile.FullName);
}
}
foreach (var rqQueryFile in exampleDir.GetFiles("*.rq"))
{
Console.WriteLine("query file: " + rqQueryFile);
var outputFile = rqQueryFile.FullName + "expected results.xml";
SparqlResultSet sparqlResultSet = null;
// try
var query = rqQueryFile.OpenText().ReadToEnd();
SparqlQuery.SparqlClasses.Query.SparqlQuery sparqlQuery = null;
{
//Perfomance.ComputeTime(() =>
{
sparqlQuery = SparqlQueryParser.Parse(store, query);
} //, exampleDir.Name+" "+rqQueryFile.Name+" parse ", true);
if (sparqlQuery != null)
// Perfomance.ComputeTime(() =>
{
sparqlResultSet = sparqlQuery.Run();
File.WriteAllText(outputFile, sparqlResultSet.ToXml().ToString());
} //, exampleDir.Name + " " + rqQueryFile.Name + " run ", true);
//var exprectedResults= SparqlResultSet.FromXml(XElement.Load(), )
Assert.AreEqual(true, true, rqQueryFile.Name);
//File.ReadAllText(rqQueryFile.FullName + " expected results.txt"),
// File.ReadAllText(outputFile));
}
// catch (Exception e)
{
// Assert.(e.Message);
}
}
}
}
}
public static void OneLUMB(IStore store, int i, int count)
{
SparqlQueryParser.Parse(store, sq5);
store.Warmup();
{
var file = new FileInfo(string.Format(@"..\..\..\Testing\examples\lubm\q ({0}).rq", i));
var q = File.ReadAllText(file.FullName);
double min = int.MaxValue, max = -1, average = 0;
double averageParse = 0;
double averageCLS = 0;
double averageRun = 0;
for (int j = 0; j < count; j++)
{
Stopwatch timer = new Stopwatch();
timer.Restart();
var sparqlQuery = SparqlQueryParser.Parse(store, q);
timer.Stop();
double time1 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageParse += (double)((int)(100 * time1 / count)) / 100;
timer.Restart();
var sparqlResultSet = sparqlQuery.Run();
timer.Stop();
double time2 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageCLS += (double)((int)(100 * time2 / count)) / 100;
timer.Restart();
sparqlResultSet.Results.ToArray();
timer.Stop();
double time3 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageRun += (double)((int)(100 * time3 / count)) / 100;
var time = time1 + time2 + time3;
average += (double)((int)(100 * time)) / 100;
if (time > max)
{
max = time;
}
if (min > time)
{
min = time;
}
File.WriteAllText(@"..\..\..\Testing\examples\lubm\q ({0}).json", sparqlResultSet.ToJson());
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine(DateTimeOffset.Now);
r.WriteLine("q " + i);
r.WriteLine("average " + average / count);
r.WriteLine("qps " + ((double)((int)(100000 * count / average)) / 100));
r.WriteLine("min " + min);
r.WriteLine("max " + max);
r.WriteLine("memory usage (bytes)" + GC.GetTotalMemory(false));
r.WriteLine("parse (ms)" + averageParse);
r.WriteLine("create linq stack " + averageCLS);
r.WriteLine("run " + averageRun);
}
}
}
public static void OneBerlinParametrized(IStore store, int i, int count)
{
SparqlQueryParser.Parse(store, sq5);
store.Warmup();
using (StreamReader streamQueryParameters = new StreamReader(string.Format(
@"..\..\..\Testing\examples\bsbm\queries\parameters\param values for{0}m {1} query.txt", 1, i)))
{
var file =
new FileInfo(string.Format(@"..\..\..\Testing\examples\bsbm\queries\parameters\{0}.rq", i));
var parametred = File.ReadAllText(file.FullName);
double min = int.MaxValue, max = -1, average = 0;
double averageParse = 0;
double averageCLS = 0;
double averageRun = 0;
for (int j = 0; j < count; j++)
{
var consted = BSBmParams.QueryReadNewParameters(parametred, streamQueryParameters);
Stopwatch timer = new Stopwatch();
timer.Restart();
var sparqlQuery = SparqlQueryParser.Parse(store, consted);
timer.Stop();
double time1 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageParse += (double)((int)(100 * time1 / count)) / 100;
timer.Restart();
var sparqlResultSet = sparqlQuery.Run();
timer.Stop();
double time2 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageCLS += (double)((int)(100 * time2 / count)) / 100;
timer.Restart();
sparqlResultSet.Results.ToArray();
timer.Stop();
double time3 = SparqlTesting.GetTimeWthLast2Digits(timer);
averageRun += (double)((int)(100 * time3 / count)) / 100;
var time = time1 + time2 + time3;
average += (double)((int)(100 * time)) / 100;
if (time > max)
{
max = time;
}
if (min > time)
{
min = time;
}
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine(DateTimeOffset.Now);
r.WriteLine("q " + i);
r.WriteLine("average " + average / count);
r.WriteLine("qps " + ((double)((int)(100000 * count / average)) / 100));
r.WriteLine("min " + min);
r.WriteLine("max " + max);
r.WriteLine("memory usage (bytes)" + GC.GetTotalMemory(false));
r.WriteLine("parse (ms)" + averageParse);
r.WriteLine("create linq stack " + averageCLS);
r.WriteLine("run " + averageRun);
}
}
}
private static void SubTestRun(StreamReader streamQueryParameters, int i1, IStore Store)
{
double[] results = new double[12];
double[] minimums = Enumerable.Repeat(double.MaxValue, 12).ToArray();
double[] maximums = new double[12];
double maxMemoryUsage = 0;
double[] totalparseMS = new double[12];
double[] totalrun = new double[12];
for (int j = 0; j < i1; j++)
{
for (int i = 0; i < 12; i++)
{
string file = string.Format(@"..\..\examples\bsbm\queries\parameters\{0}.rq", i + 1);
var readAllText = File.ReadAllText(file);
readAllText = BSBmParams.QueryReadParameters(readAllText, streamQueryParameters);
var timer = new Stopwatch();
SparqlQuery.SparqlClasses.Query.SparqlQuery sparqlQuery = null;
// if (i == 0)
{
sparqlQuery = SparqlQueryParser.Parse(Store, readAllText);
}
totalparseMS[i] += GetTimeWthLast2Digits(timer);
var st1 = DateTime.Now;
//if (i == 0)
{
var sparqlResultSet = sparqlQuery.Run().ToJson();
}
totalrun[i] += (DateTime.Now - st1).Ticks / 10000L;
var totalMilliseconds = GetTimeWthLast2Digits(timer);
var memoryUsage = GC.GetTotalMemory(false);
if (memoryUsage > maxMemoryUsage)
{
maxMemoryUsage = memoryUsage;
}
if (minimums[i] > totalMilliseconds)
{
minimums[i] = totalMilliseconds;
}
if (maximums[i] < totalMilliseconds)
{
maximums[i] = totalMilliseconds;
}
results[i] += totalMilliseconds;
// File.WriteAllText(Path.ChangeExtension(file.FullName, ".txt"), resultString);
//.Save(Path.ChangeExtension(file.FullName,".xml"));
}
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine("milions " + Millions);
r.WriteLine("date time " + DateTime.Now);
r.WriteLine("max memory usage " + maxMemoryUsage);
r.WriteLine("average " + string.Join(", ", results.Select(l => l == 0 ? "inf" : (500 * 1000 / l).ToString())));
r.WriteLine("minimums " + string.Join(", ", minimums));
r.WriteLine("maximums " + string.Join(", ", maximums));
r.WriteLine("total parse " + string.Join(", ", totalparseMS));
r.WriteLine("total run " + string.Join(", ", totalrun));
r.WriteLine("total " + totalparseMS.Sum() + totalrun.Sum());
// r.WriteLine("countCodingUsages {0} totalMillisecondsCodingUsages {1}", TripleInt.EntitiesCodeCache.Count, TripleInt.totalMilisecondsCodingUsages);
//r.WriteLine("EWT average search" + EntitiesMemoryHashTable.total / EntitiesMemoryHashTable.count);
//r.WriteLine("EWT average range" + EntitiesMemoryHashTable.totalRange / EntitiesMemoryHashTable.count);
}
}