public TracingTripleStoreAbstractInt(string path, RDFIntStoreAbstract @base) : base(@base.EntityCoding, @base.PredicatesCoding, @base.NameSpaceStore, @base.LiteralStore, @base.Scale)
{
this.@base = @base;
xPath = path + "tracing.xml";
if (File.Exists(xPath))
{
// File.Delete(xPath);
}
x = new XElement("tracing");
}
private static void RunBerlinsParameters(RDFIntStoreAbstract ts)
{
Console.WriteLine("antrl parametered");
var fileInfos = new[]
{
@"..\..\sparql data\queries\parameters\1.rq",
@"..\..\sparql data\queries\parameters\2.rq",
@"..\..\sparql data\queries\parameters\3.rq",
@"..\..\sparql data\queries\parameters\4.rq",
@"..\..\sparql data\queries\parameters\5.rq",
@"..\..\sparql data\queries\parameters\6.rq",
@"..\..\sparql data\queries\parameters\7.rq",
@"..\..\sparql data\queries\parameters\8.rq",
@"..\..\sparql data\queries\parameters\9.rq",
@"..\..\sparql data\queries\parameters\10.rq",
@"..\..\sparql data\queries\parameters\11.rq",
@"..\..\sparql data\queries\parameters\12.rq",
}
.Select(s => new FileInfo(s))
.ToArray();
var paramvaluesFilePath = string.Format(@"..\..\sparql data\queries\parameters\param values for{0} m.txt", Millions);
// using (StreamWriter streamQueryParameters = new StreamWriter(paramvaluesFilePath))
// for (int j = 0; j < 1000; j++)
// foreach (var file in fileInfos.Select(info => File.ReadAllText(info.FullName)))
// QueryWriteParameters(file, streamQueryParameters, ts);
//return;
using (StreamReader streamQueryParameters = new StreamReader(paramvaluesFilePath))
{
for (int j = 0; j < 500; j++)
{
fileInfos.Select(file => QueryReadParameters(File.ReadAllText(file.FullName),
streamQueryParameters))
.Select(queryReadParameters =>
{
return("");
// var q = new Query(ts); q.Parse(queryReadParameters, ts); return q.Run();
})
.ToArray();
}
SubTestRun(ts, fileInfos, streamQueryParameters, 500);
}
}
private static void QueryWriteParameters(string parameteredQuery, StreamWriter output, RDFIntStoreAbstract ts)
{
var productsCodes = ts.GetSubjectByObjPred(
ts.CodeEntityFullName("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/Product>"),
ts.CodePredicateFullName("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>"));
var codes = productsCodes as int[] ?? productsCodes.ToArray();
int productCount = codes.Count();
var product = ts.DecodeEntityFullName(codes.ElementAt(random.Next(0, productCount)));
//Millions == 1000 ? 2855260 : Millions == 100 ? 284826 : Millions == 10 ? 284826 : 2785;
int productFeatureCount =
Millions == 1000 ? 478840 : Millions == 100 ? 47884 : Millions == 10 ? 47450 : 4745;
int productTypesCount = Millions == 1000 ? 20110 : Millions == 100 ? 2011 : Millions == 10 ? 1510 : 151;
//var review = random.Next(1, productCount*10);
////var product = random.Next(1, productCount);
////var productProducer = product/ProductsPerProducer + 1;
//var offer = random.Next(1, productCount*OffersPerProduct);
//var vendor = offer/OffersPerVendor + 1;
var offersCodes = ts.GetSubjectByObjPred(
ts.CodeEntityFullName("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/Offer>"),
ts.CodePredicateFullName("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>"));
codes = offersCodes as int[] ?? offersCodes.ToArray();
var offer = ts.DecodeEntityFullName(codes[random.Next(0, codes.Length)]);
var reviewsCodes = ts.GetSubjectByObjPred(
ts.CodeEntityFullName("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/Review>"),
ts.CodePredicateFullName("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>"));
codes = reviewsCodes as int[] ?? reviewsCodes.ToArray();
var review = ts.DecodePredicateFullName(codes[random.Next(0, codes.Length)]);
if (parameteredQuery.Contains("%ProductType%"))
{
output.WriteLine("bsbm-inst:ProductType" + random.Next(1, productTypesCount));
}
if (parameteredQuery.Contains("%ProductFeature1%"))
{
output.WriteLine("bsbm-inst:ProductFeature" + random.Next(1, productFeatureCount));
}
if (parameteredQuery.Contains("%ProductFeature2%"))
{
output.WriteLine("bsbm-inst:ProductFeature" + random.Next(1, productFeatureCount));
}
if (parameteredQuery.Contains("%ProductFeature3%"))
{
output.WriteLine("bsbm-inst:ProductFeature" + random.Next(1, productFeatureCount));
}
if (parameteredQuery.Contains("%x%"))
{
output.WriteLine(random.Next(1, 500).ToString());
}
if (parameteredQuery.Contains("%y%"))
{
output.WriteLine(random.Next(1, 500).ToString());
}
if (parameteredQuery.Contains("%ProductXYZ%"))
{
output.WriteLine(product);//"<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer{0}/Product{1}>",productProducer, product);
}
if (parameteredQuery.Contains("%word1%"))
{
output.WriteLine(words[random.Next(0, words.Length)]);
}
if (parameteredQuery.Contains("%currentDate%"))
{
output.WriteLine("\"" + DateTime.Today.AddYears(-6) + "\"^^<http://www.w3.org/2001/XMLSchema#dateTime>");
}
if (parameteredQuery.Contains("%ReviewXYZ%"))
{
output.WriteLine(review);//"<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromRatingSite{0}/Review{1}>",review/10000 + 1, review);
}
if (parameteredQuery.Contains("%OfferXYZ%"))
{
output.WriteLine(offer);
}
//"<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromVendor{0}/Offer{1}>", vendor, offer);
}
private static void RunBerlinsWithConstants(RDFIntStoreAbstract ts)
{
long[] results = new long[12];
Console.WriteLine("antrl with constants");
int i = 0;
var fileInfos = new[]
{
@"..\..\sparql data\queries\with constants\1.rq",
@"..\..\sparql data\queries\with constants\2.rq",
@"..\..\sparql data\queries\with constants\3.rq",
@"..\..\sparql data\queries\with constants\4.rq",
@"..\..\sparql data\queries\with constants\5.rq",
@"..\..\sparql data\queries\with constants\6.rq",
@"..\..\sparql data\queries\with constants\7.rq",
@"..\..\sparql data\queries\with constants\8.rq",
@"..\..\sparql data\queries\with constants\9.rq",
@"..\..\sparql data\queries\with constants\10.rq",
@"..\..\sparql data\queries\with constants\11.rq",
@"..\..\sparql data\queries\with constants\12.rq",
}
.Select(s => new FileInfo(s))
.ToArray();
for (int j = 0; j < 0; j++)
{
foreach (var file in fileInfos)
{
var readAllText = File.ReadAllText(file.FullName);
// var st = DateTime.Now;
// var q = new Query(ts);
// q.Parse(readAllText, ts);
// var resultString = q.Run();
//var totalMilliseconds = (long)(DateTime.Now - st).TotalMilliseconds;
// results[i++] += totalMilliseconds;
// File.WriteAllText(Path.ChangeExtension(file.FullName, ".txt"), resultString);
//.Save(Path.ChangeExtension(file.FullName,".xml"));
}
}
for (int j = 0; j < 1; j++)
{
i = 0;
foreach (var file in fileInfos)
{
var readAllText = File.ReadAllText(file.FullName);
var st = DateTime.Now;
// var q = new Query(ts);
// q.Parse(readAllText, ts);
var resultString = "";//q.Run();
var totalMilliseconds = (DateTime.Now - st).Ticks / 10000L;
results[i++] += totalMilliseconds;
File.WriteAllText(Path.ChangeExtension(file.FullName, ".txt"), resultString);
//.Save(Path.ChangeExtension(file.FullName,".xml"));
}
}
Console.WriteLine(string.Join(", ", results));
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine("milions " + Millions);
// r.WriteLine("countCodingUsages {0} totalMillisecondsCodingUsages {1}", TripleInt.EntitiesCodeCache.Count, TripleInt.totalMilisecondsCodingUsages);
}
}
private static void SubTestRun(RDFIntStoreAbstract ts, FileInfo[] fileInfos, StreamReader streamQueryParameters, int i1)
{
int i;
long[] results = new long[12];
double[] minimums = Enumerable.Repeat(double.MaxValue, 12).ToArray();
double[] maximums = new double[12];
double maxMemoryUsage = 0;
long[] totalparseMS = new long[12];
long[] totalrun = new long[12];
for (int j = 0; j < i1; j++)
{
i = 0;
foreach (var file in fileInfos)
{
var readAllText = File.ReadAllText(file.FullName);
readAllText = QueryReadParameters(readAllText, streamQueryParameters);
var st = DateTime.Now;
// var q = new Query(ts);
// q.Parse(readAllText, ts);
totalparseMS[i] += (DateTime.Now - st).Ticks / 10000L;
var st1 = DateTime.Now;
// var resultString = //q.Run();
var totalMilliseconds = (DateTime.Now - st).Ticks / 10000L;
totalrun[i] += (DateTime.Now - st1).Ticks / 10000L;
var memoryUsage = GC.GetTotalMemory(false);
if (memoryUsage > maxMemoryUsage)
{
maxMemoryUsage = memoryUsage;
}
if (minimums[i] > totalMilliseconds)
{
minimums[i] = totalMilliseconds;
}
if (maximums[i] < totalMilliseconds)
{
maximums[i] = totalMilliseconds;
}
results[i++] += totalMilliseconds;
// File.WriteAllText(Path.ChangeExtension(file.FullName, ".txt"), resultString);
//.Save(Path.ChangeExtension(file.FullName,".xml"));
}
}
using (StreamWriter r = new StreamWriter(@"..\..\output.txt", true))
{
r.WriteLine("milions " + Millions);
r.WriteLine("max memory usage " + maxMemoryUsage);
r.WriteLine("average " + string.Join(", ", results.Select(l => l == 0 ? "inf" : (500 * 1000 / l).ToString())));
r.WriteLine("minimums " + string.Join(", ", minimums));
r.WriteLine("maximums " + string.Join(", ", maximums));
r.WriteLine("total parse " + string.Join(", ", totalparseMS));
r.WriteLine("total run " + string.Join(", ", totalrun));
// r.WriteLine("countCodingUsages {0} totalMillisecondsCodingUsages {1}", TripleInt.EntitiesCodeCache.Count, TripleInt.totalMilisecondsCodingUsages);
//r.WriteLine("EWT average search" + EntitiesMemoryHashTable.total / EntitiesMemoryHashTable.count);
//r.WriteLine("EWT average range" + EntitiesMemoryHashTable.totalRange / EntitiesMemoryHashTable.count);
}
}
private static IEnumerable <TripletGraph> LoadGraphs(string datafile, RDFIntStoreAbstract rdfIntStore)
{
int ntriples = 0;
int nTripletsInBuffer = 0;
string subject = null;
List <TripletGraph> bufferTripletsGrpah = new List <TripletGraph>(BufferMax);
TripletGraph currentTripletGraph = null;
HashSet <string> entitiesStrings = new HashSet <string>();
using (var sr = new StreamReader(datafile))
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
//if (i % 10000 == 0) { Console.Write("{0} ", i / 10000); }
if (line == "")
{
continue;
}
if (line[0] == '@')
{
// namespace
string[] parts = line.Split(' ');
if (parts.Length != 4 || parts[0] != "@prefix" || parts[3] != ".")
{
Console.WriteLine("Err: strange line: " + line);
continue;
}
string pref = parts[1];
string nsname = parts[2];
if (nsname.Length < 3 || nsname[0] != '<' || nsname[nsname.Length - 1] != '>')
{
Console.WriteLine("Err: strange nsname: " + nsname);
continue;
}
nsname = nsname.Substring(1, nsname.Length - 2);
rdfIntStore.NameSpaceStore.AddPrefix(pref, nsname);
}
else if (line[0] != ' ')
{
//if (bufferTripletsGrpah.Count >= BufferMax)
if (nTripletsInBuffer >= BufferMax)
{
rdfIntStore.EntityCoding.InsertPortion(entitiesStrings);
foreach (var tripletGraph in bufferTripletsGrpah)
{
yield return(tripletGraph);
}
bufferTripletsGrpah.Clear();
entitiesStrings.Clear();
GC.Collect();
nTripletsInBuffer = 0;
}
// Subject
line = line.Trim();
subject = rdfIntStore.NameSpaceStore.GetShortFromFullOrPrefixed(line);
entitiesStrings.Add(subject);
currentTripletGraph = new TripletGraph()
{
subject = subject
};
bufferTripletsGrpah.Add(currentTripletGraph);
}
else
{
// Predicate and object
string line1 = line.Trim();
int first_blank = line1.IndexOf(' ');
if (first_blank == -1)
{
Console.WriteLine("Err in line: " + line);
continue;
}
string rest_line = line1.Substring(first_blank + 1).Trim();
// Óáåðåì ïîñëåäíèé ñèìâîë
rest_line = rest_line.Substring(0, rest_line.Length - 1).Trim();
bool isDatatype = rest_line[0] == '\"';
string pred_line = line1.Substring(0, first_blank);
string predicateString = rdfIntStore.NameSpaceStore.GetShortFromFullOrPrefixed(pred_line);
// îáúåêò ìîæåò áûòü entity èëè äàííîå, ó äàííîãî ìîæåò áûòü ÿçûêîâûé ñïåöèôèêàòîð èëè òèï
string sdata = null;
string datatype = null;
string lang = null;
if (isDatatype)
{
// Ïîñëåäíÿÿ äâîéíàÿ êàâû÷êà
int lastqu = rest_line.LastIndexOf('\"');
// Çíà÷åíèå äàííûõ
sdata = rest_line.Substring(1, lastqu - 1);
// ßçûêîâûé ñïåöèàëèçàòîð:
int dog = rest_line.LastIndexOf('@');
if (dog == lastqu + 1)
{
lang = rest_line.Substring(dog + 1, rest_line.Length - dog - 1);
}
int pp = rest_line.IndexOf("^^");
if (pp == lastqu + 1)
{
// Òèï äàííûõ
string qname = rest_line.Substring(pp + 2);
// òèï äàííûõ ìîæåò áûòü "ïðåôèêñíûì" èëè ïîëíûì
//datatype = qname[0] == '<'
// ? qname.Substring(1, qname.Length - 2)
// : GetEntityString(namespaces, qname);
datatype = rdfIntStore.NameSpaceStore.GetShortFromFullOrPrefixed(qname);
}
Literal literal = rdfIntStore.LiteralStore.Create(datatype, sdata, lang);
rdfIntStore.PredicatesCoding.Insert(predicateString, literal.vid);
currentTripletGraph.PredicateDataValuePairs.Add(
new KeyValuePair <int, Literal>(rdfIntStore.PredicatesCoding[predicateString],
rdfIntStore.LiteralStore.Write(literal)));
}
else
{
rdfIntStore.PredicatesCoding.Insert(predicateString, null);
string obj = rdfIntStore.NameSpaceStore.GetShortFromFullOrPrefixed(rest_line);
entitiesStrings.Add(obj);
currentTripletGraph.PredicateObjValuePairs.Add(new KeyValuePair <int, string>(rdfIntStore.PredicatesCoding[predicateString], obj));
}
ntriples++;
nTripletsInBuffer++;
if (ntriples % 100000 == 0)
{
Console.Write("r{0} ", ntriples / 100000);
}
}
}
rdfIntStore.EntityCoding.InsertPortion(entitiesStrings);
foreach (var tripletGraph in bufferTripletsGrpah)
{
yield return(tripletGraph);
}
bufferTripletsGrpah.Clear();
rdfIntStore.MakeIndexed();
entitiesStrings.Clear();
GC.Collect();
Console.WriteLine("ntriples={0}", ntriples);
}
private static IEnumerable <TripleInt> LoadTriplets(string datafile, RDFIntStoreAbstract rdf)
{
int ntriples = 0;
int nTripletsInBuffer = 0;
string subject = null;
int subjectCode = 0;
using (var sr = new StreamReader(datafile))
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
//if (i % 10000 == 0) { Console.Write("{0} ", i / 10000); }
if (line == "")
{
continue;
}
if (line[0] == '@')
{
// namespace
string[] parts = line.Split(' ');
if (parts.Length != 4 || parts[0] != "@prefix" || parts[3] != ".")
{
Console.WriteLine("Err: strange line: " + line);
continue;
}
string pref = parts[1];
string nsname = parts[2];
if (nsname.Length < 3 || nsname[0] != '<' || nsname[nsname.Length - 1] != '>')
{
Console.WriteLine("Err: strange nsname: " + nsname);
continue;
}
nsname = nsname.Substring(1, nsname.Length - 2);
rdf.NameSpaceStore.namespacesByPrefix.Add(pref, nsname);
string ns = nsname;
// @namespace = @namespace.ToLower();
if (ns[ns.Length - 1] == '/' || ns[ns.Length - 1] == '\\' ||
ns[ns.Length - 1] == '#')
{
ns = ns.Substring(0, ns.Length - 1);
}
int code;
if (!rdf.NameSpaceStore.Codes.TryGetValue(ns, out code))
{
rdf.NameSpaceStore.Codes.Add(ns, code = rdf.NameSpaceStore.Codes.Count);
rdf.NameSpaceStore.NameSpaceStrings.Add(ns);
}
int temp = code;
}
else if (line[0] != ' ')
{
// Subject
line = line.Trim();
subject = rdf.NameSpaceStore.GetShortFromFullOrPrefixed(line);
subjectCode = rdf.EntityCoding.InsertOne(subject);
}
else
{
// Predicate and object
string line1 = line.Trim();
int first_blank = line1.IndexOf(' ');
if (first_blank == -1)
{
Console.WriteLine("Err in line: " + line);
continue;
}
string rest_line = line1.Substring(first_blank + 1).Trim();
// Уберем последний символ
rest_line = rest_line.Substring(0, rest_line.Length - 1).Trim();
bool isDatatype = rest_line[0] == '\"';
string pred_line = line1.Substring(0, first_blank);
string predicateString = rdf.NameSpaceStore.GetShortFromFullOrPrefixed(pred_line);
// объект может быть entity или данное, у данного может быть языковый спецификатор или тип
string sdata = null;
string datatype = null;
string lang = null;
if (isDatatype)
{
// Последняя двойная кавычка
int lastqu = rest_line.LastIndexOf('\"');
// Значение данных
sdata = rest_line.Substring(1, lastqu - 1);
// Языковый специализатор:
int dog = rest_line.LastIndexOf('@');
if (dog == lastqu + 1)
{
lang = rest_line.Substring(dog + 1, rest_line.Length - dog - 1);
}
int pp = rest_line.IndexOf("^^");
if (pp == lastqu + 1)
{
// Тип данных
string qname = rest_line.Substring(pp + 2);
// тип данных может быть "префиксным" или полным
datatype = rdf.NameSpaceStore.GetShortFromFullOrPrefixed(qname);
}
Literal literal = rdf.LiteralStore.Create(datatype, sdata, lang);
rdf.PredicatesCoding.Insert(predicateString, literal.vid);
yield return(new DTripleInt(subjectCode, rdf.PredicatesCoding[predicateString],
rdf.LiteralStore.Write(literal)));
}
else
{
rdf.PredicatesCoding.Insert(predicateString, null);
string obj = rdf.NameSpaceStore.GetShortFromFullOrPrefixed(rest_line);
yield return
(new OTripleInt(subjectCode, rdf.PredicatesCoding[predicateString],
rdf.EntityCoding.InsertOne(obj)));
}
ntriples++;
nTripletsInBuffer++;
if (ntriples % 100000 == 0)
{
Console.Write("r{0} ", ntriples / 100000);
}
}
}
rdf.MakeIndexed();
GC.Collect();
Console.WriteLine("ntriples={0}", ntriples);
}
public static void LoadTriplets(string filepath, PaCell otriples, PaCell dtriplets, RDFIntStoreAbstract rdf)
{
//Directory.Delete(path);
//Directory.CreateDirectory(path);
otriples.Clear();
otriples.Fill(new object[0]);
dtriplets.Clear();
dtriplets.Fill(new object[0]);
int i = 0;
//Entity e = new Entity();
rdf.Clear();
foreach (var triplet in LoadTriplets(filepath, rdf))
{
if (i % 100000 == 0)
{
Console.Write("w{0} ", i / 100000);
}
i++;
if (triplet is OTripleInt)
{
var oTripleInt = triplet as OTripleInt;
otriples.Root.AppendElement(new object[]
{
oTripleInt.subject,
oTripleInt.predicate,
oTripleInt.obj
});
}
else
{
var dtr = triplet as DTripleInt;
dtriplets.Root.AppendElement(new object[]
{
dtr.subject,
dtr.predicate,
dtr.data.Offset
});
}
}
otriples.Flush();
dtriplets.Flush();
}
public static void LoadByGraphsBuffer(string filepath, PaCell otriples, PaCell dtriplets, RDFIntStoreAbstract rdfIntStore)
{
//Directory.Delete(path);
//Directory.CreateDirectory(path);
otriples.Clear();
otriples.Fill(new object[0]);
dtriplets.Clear();
dtriplets.Fill(new object[0]);
int i = 0;
//Entity e = new Entity();
rdfIntStore.Clear();
foreach (var tripletGrpah in LoadGraphs(filepath, rdfIntStore))
{
if (i % 100000 == 0)
{
Console.Write("w{0} ", i / 100000);
}
i += tripletGrpah.PredicateDataValuePairs.Count + tripletGrpah.PredicateObjValuePairs.Count;
var subject = rdfIntStore.EntityCoding.GetCode(tripletGrpah.subject);
foreach (var predicateObjValuePair in tripletGrpah.PredicateObjValuePairs)
{
otriples.Root.AppendElement(new object[]
{
subject,
predicateObjValuePair.Key,
rdfIntStore.EntityCoding.GetCode(predicateObjValuePair.Value)
});
}
foreach (var predicateDataValuePair in tripletGrpah.PredicateDataValuePairs)
{
dtriplets.Root.AppendElement(new object[]
{
subject,
predicateDataValuePair.Key,
predicateDataValuePair.Value.Offset
});
}
}
otriples.Flush();
dtriplets.Flush();
rdfIntStore.NameSpaceStore.Flush();
rdfIntStore.LiteralStore.Flush();
}
