protected RDFIntStoreAbstract(string path, IStringIntCoding entityCoding, PredicatesCoding predicatesCoding,
NameSpaceStore nameSpaceStore, LiteralStoreAbstract literalStore)
: this(entityCoding, predicatesCoding, nameSpaceStore, literalStore, new ScaleCell(path))
{
// TODO: Complete member initialization
this.path = path;
}
public virtual void MakeIndexed()
{
EntityCoding.MakeIndexed();
PredicatesCoding.MakeIndexed();
NameSpaceStore.Flush();
Scale.Flush();
LiteralStore.Flush();
}
public virtual void Clear()
{
LiteralStore.Clear();
EntityCoding.Clear();
PredicatesCoding.Clear();
NameSpaceStore.Clear();
LiteralStore.InitConstants(NameSpaceStore);
Scale.Clear();
}
protected RDFIntStoreAbstract(IStringIntCoding entityCoding, PredicatesCoding predicatesCoding, NameSpaceStore nameSpaceStore, LiteralStoreAbstract literalStore,
ScaleCell scale)
{
this.literalStore = literalStore;
this.entityCoding = entityCoding;
this.nameSpaceStore = nameSpaceStore;
this.predicatesCoding = predicatesCoding;
this.scale = scale;
if (!scale.Cell.IsEmpty)
{
scale.CalculateRange();
}
}
//// Идея хорошая, но надо менять схему реализации
//private GroupedEntities getable;
//private Dictionary<int, object[]> geHash;
public ColumnsStore(string path, IStringIntCoding entityCoding, PredicatesCoding predicatesCoding, NameSpaceStore nameSpaceStore, LiteralStoreAbstract literalStore)
: base(path, entityCoding, predicatesCoding, nameSpaceStore, literalStore)
{
InitTypes();
otriplets_op_filePath = path + "otriples_op.pac";
otriples_filePath = path + "otriples.pac";
dtriples_filePath = path + "dtriples.pac";
dataPredicatesColumn_filePath = path + "dataPredicatesColumn.pac";
objPredicatesColumn_filePath = path + "objPredicatesColumn.pac";
invPredicatesColumn_filePath = path + "invPredicatesColumn.pac";
invSubjectsColumn_filePath = path + "invSubjectsColumn.pac";
objectsColumn_filePath = path + "objectsColumn.pac";
dataColumn_filePath = path + "dataColumn.pac";
Open(File.Exists(otriples_filePath));
ewt = new EntitiesWideTable(path, 3);
//ewtHash = new EntitiesMemoryHashTable(ewt);
// ewtHash.Load();
//getable = new GroupedEntities(path); // Это хорошая идея, но нужно менять схему реализации
//getable.CheckGroupedEntities();
//geHash = getable.GroupedEntitiesHash();
}
//// Идея хорошая, но надо менять схему реализации
//private GroupedEntities getable;
//private Dictionary<int, object[]> geHash;
public TripleStoreInt(string path, IStringIntCoding entityCoding, PredicatesCoding predicatesCoding, NameSpaceStore nameSpaceStore, LiteralStoreAbstract literalStore)
: base(path, entityCoding, predicatesCoding, nameSpaceStore, literalStore)
{
this.path = path;
InitTypes();
otriplets_op_filePath = path + "otriples_op.pac";
otriples_filePath = path + "otriples.pac";
dtriples_filePath = path + "dtriples_spf.pac";
if (File.Exists(otriples_filePath))
{
Open(true);
}
else
{
otriples = new PaCell(tp_triple_seq_two, otriples_filePath, false);
otriples_op = new PaCell(tp_triple_seq_two, otriplets_op_filePath, false);
dtriples_sp = new PaCell(tp_dtriple_spf_two, dtriples_filePath, false);
}
if (!Scale.Cell.IsEmpty)
{
Scale.CalculateRange();
}
ewt = new EntitiesWideTable(path, 3);
//ewtHash = new EntitiesMemoryHashTable(ewt);
// ewtHash.Load();
//getable = new GroupedEntities(path); // Это хорошая идея, но нужно менять схему реализации
//getable.CheckGroupedEntities();
//geHash = getable.GroupedEntitiesHash();
}
private static void Test()
{
string path = (@"C:\Users\Admin\Source\Repos\PolarDemo\Databases\" + Millions + @"mln\");
var nameSpaceStore = new NameSpaceStore(path);
StringIntMD5RAMUnsafe stringIntMd5RamUnsafe = new StringIntMD5RAMUnsafe(path);
PredicatesCoding predicatesCoding = new PredicatesCoding(path);
RDFIntStoreAbstract ts = new CashingTripleStoreInt(
new RdfTrees(path,
// new CasheCoding(new StringIntCoding(path+"entitiesCodes")),
stringIntMd5RamUnsafe,
predicatesCoding,
nameSpaceStore,
new LiteralStore(path, nameSpaceStore)));
// bool load = false;
bool load = false;
using (StreamWriter wr = new StreamWriter(@"..\..\output.txt", true))
wr.WriteLine("millions " + Millions);
DateTime start = DateTime.Now;
long spent = 0;
if (load)
{
ts.LoadTurtle(@"C:\deployed\" + Millions + "M.ttl", useBuffer: false);
}
else
{
var obj2 = stringIntMd5RamUnsafe.GetCode(nameSpaceStore.GetShortFromFullOrPrefixed("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature19>"));
var obj1 = stringIntMd5RamUnsafe.GetCode(nameSpaceStore.GetShortFromFullOrPrefixed("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature8>"));
var pred = predicatesCoding.GetCode(nameSpaceStore.GetShortFromFullOrPrefixed("<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productFeature>"));
var triplets1 = ts.GetSubjectByObjPred(obj1, pred);
var triplets2 = ts.GetSubjectByObjPred(obj2, pred);
foreach (var i in triplets1.Where(i => triplets2.Contains(i)))
{
Console.WriteLine("sdfg");
}
for (int i = 0; i < stringIntMd5RamUnsafe.Count; i++)
{
if (
ts.GetObjBySubj(i)
.Any(
keyValuePair =>
!ts.GetSubjectByObjPred(keyValuePair.Key, keyValuePair.Value).Contains(i)))
{
throw new Exception();
}
}
return;
ts.WarmUp();
spent = (DateTime.Now - start).Ticks / 10000;
using (StreamWriter wr = new StreamWriter(@"..\..\output.txt", true))
wr.WriteLine("warm up " + spent + " мс.");
RunBerlinsWithConstants(ts);
// RunBerlinsParameters(ts);
}
spent = (DateTime.Now - start).Ticks / 10000;
using (StreamWriter wr = new StreamWriter(@"..\..\output.txt", true))
wr.WriteLine("total " + spent + " мс.");
// (ts as TracingTripleStoreInt).x.Save((ts as TracingTripleStoreInt).xPath);
}
public int CodeEntityFullOrShort(string name)
{
return(EntityCoding.GetCode(NameSpaceStore.GetShortFromFullOrPrefixed(name)));
}
public int CodePredicateFullOrShort(string name)
{
return(PredicatesCoding.GetCode(NameSpaceStore.GetShortFromFullOrPrefixed(name)));
}
public int CodeEntityFullName(string name)
{
return(EntityCoding.GetCode(NameSpaceStore.FromFullName(name.Substring(1, name.Length - 2))));
}
public int CodePredicateFullName(string name)
{
return(PredicatesCoding.GetCode(NameSpaceStore.FromFullName(name.Substring(1, name.Length - 2))));
}
public string DecodePredicateFullName(int code)
{
return(NameSpaceStore.DecodeNsShortName(PredicatesCoding.GetName(code)));
}
public string DecodeEntityFullName(int code)
{
return(NameSpaceStore.DecodeNsShortName(EntityCoding.GetName(code)));
}
/// <summary>
/// Конструктор
/// </summary>
/// <param name="path">директория базы данных с (обратным) слешем</param>
/// <param name="literalStore"></param>
/// <param name="entityCoding"></param>
/// <param name="nameSpaceStore"></param>
/// <param name="predicatesCoding"></param>
public RdfTrees(string path, IStringIntCoding entityCoding, PredicatesCoding predicatesCoding, NameSpaceStore nameSpaceStore, LiteralStoreAbstract literalStore)
: base(path, entityCoding, predicatesCoding, nameSpaceStore, literalStore)
{
// Построим типы
InitTypes();
// Создадим или откроем ячейки
this.entitiesTree = new PxCell(tp_entitiesTree, entitiesTreePath = path + "entitiesTree.pxc", false);
//this.literalsTree = new PxCell(tp_literalsTree, path + "literalsTree.pxc", false);
// this.dtriples = new PaCell(tp_dtriple_spf, path + "dtriples.pac", false); // Это вместо не работающего дерева литералов }
otriples = new PaCell(tp_otriple_seq, path + "otriples.pac", File.Exists(path + "otriples.pac"));
otriples.Close();
}
public static void Main(string[] args)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
string[] ids = new string[]
{
//"svet_100616111408_10844",
//"pavl_100531115859_2020",
//"piu_200809051791",
//"pavl_100531115859_6952",
//"svet_100616111408_10864",
//"w20090506_svetlana_5727",
//"piu_200809051742",
//"p0013313",
//"p0011098",
//"svet_100616111408_14354"
//"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature19",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature13",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature4",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature8",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature11",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature3",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature19",
"http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/Product",
};
DateTime tt0 = DateTime.Now;
Console.WriteLine("Start");
string path = "../../../Databases/";
//TripleStore<EntityS> ts = new TripleStore<EntityS>(path, new PolarDB.PType(PolarDB.PTypeEnumeration.sstring));
NameSpaceStore nameSpaceStore = new NameSpaceStore(path);
TripleStoreInt ts = new TripleStoreInt(path, new StringIntMD5RAMCollision(path), new PredicatesCoding(path), nameSpaceStore, new LiteralStore(path, nameSpaceStore));
Console.WriteLine("InitTripleStore duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
bool toload = true;
if (toload)
{
//ts.LoadXML(path + "0001.xml");
//Console.WriteLine("LoadXML ok.");
PolarDB.PaEntry.bufferBytes = 1000000000; //2*1000*1000*1000;
//ts.LoadTurtle(@"D:\deployed\dataset_100M.ttl");
ts.LoadTurtle(@"D:\home\FactographDatabases\dataset\dataset10m.ttl", true);
Console.WriteLine("LoadTurtle ok.");
Console.WriteLine("duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
return;
}
ts.WarmUp();
Console.WriteLine("WarmUp duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
bool run148q5 = true;
if (run148q5)
{
int cnt = 500;//BerlinTestsInt.sarr.Count();
long dur;
DateTime tt00 = DateTime.Now;
bool secondtest = true;
if (secondtest)
{
foreach (var sprod in BerlinTestsInt.sarr)
{
var query = BerlinTestsInt.Query2param(ts, sprod);
Console.WriteLine("22222 {0} d={1}", query.Count(), (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
}
dur = (DateTime.Now - tt00).Ticks / 10000L;
Console.WriteLine("Total time for {0} queries: {1}. Everage: {2}. QpS: {3}",
cnt, dur, (double)dur / (double)cnt, cnt * 1000 / dur);
tt00 = DateTime.Now;
}
bool fifthtest = false;
if (fifthtest)
{
foreach (var sprod in Allproducts.Products.Take(2000))
{
var query = BerlinTestsInt.Query5parameter(ts, sprod);
query.Count();
}
tt00 = DateTime.Now;
foreach (var sprod in Allproducts.Products.Skip(2000).Take(cnt))
{
var query = BerlinTestsInt.Query5parameter(ts, sprod);
//var query = BerlinTestsInt.Query2param(ts, sprod);
// Console.WriteLine("55555 {0} d={1}", query.Count(), (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
query.Count();
}
dur = (DateTime.Now - tt00).Ticks / 10000L;
Console.WriteLine("Total time for {0} queries: {1}. Everage: {2}. QpS: {3}",
cnt, dur, (double)dur / (double)cnt, cnt * 1000 / dur);
tt00 = DateTime.Now;
}
//tt00 = DateTime.Now;
//foreach (var sprod in BerlinTestsInt.sarr)
//{
// var query = BerlinTestsInt.Query5parameter(ts, sprod);
// //var query = BerlinTestsInt.Query2param(ts, sprod);
// Console.WriteLine("22222 {0} d={1}", query.Count(), (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//}
//dur = (DateTime.Now - tt00).Ticks / 10000L;
//Console.WriteLine("Total time for {0} queries: {1}. Everage: {2}. QpS: {3}",
// cnt, dur, (double)dur / (double)cnt, cnt * 1000 / dur);
}
bool runpseudosoalqltests = false;
if (runpseudosoalqltests)
{
var berlin1 = BerlinTestsInt.Berlin1(ts);
tt0 = DateTime.Now;
//var query0 = BerlinTests.Query0(ts);
var query1 = BerlinTestsInt.Query1(ts);
var query2 = BerlinTestsInt.Query2(ts);
var query1_1 = BerlinTestsInt.Query1_1(ts);
var berlin3 = BerlinTestsInt.Berlin3(ts);
var query3 = BerlinTestsInt.Query3(ts);
var query5 = BerlinTestsInt.Query5(ts);
var berlin6 = BerlinTestsInt.Berlin6(ts);
var query6 = BerlinTestsInt.Query6(ts);
tt0 = DateTime.Now;
//Console.WriteLine(query3.Count());
//Console.WriteLine("query0 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//return;
//Console.WriteLine(query1_1.Count());
//Console.WriteLine("1_1 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
ts.TestsOfMethods(ids, ts);
Console.WriteLine(query1.Count());
Console.WriteLine("1 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//Console.WriteLine(berlin1.Count());
//Console.WriteLine("Berlin1 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
Console.WriteLine(query2.Count());
Console.WriteLine("2 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//Console.WriteLine(berlin3.Count());
//Console.WriteLine("berlin3 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
Console.WriteLine(query3.Count());
Console.WriteLine("3 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
Console.WriteLine(query5.Count());
Console.WriteLine("5 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//Console.WriteLine(berlin6.Count());
//Console.WriteLine("berlin6 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
Console.WriteLine(query6.Count());
Console.WriteLine("6 duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//foreach (var rw in query1)
//{
// Console.WriteLine("{0} {1}", rw.row[1], rw.row[2]);
//}
//Console.WriteLine();
//foreach (var ovr in berlin1)
//{
// Console.WriteLine("{0} {1}", ovr.row[7], ovr.row[8]);
//}
return;
}
bool pseudosparql = false;
if (pseudosparql)
{
var query = BerlinTestsInt.Query3_1(ts);
int count = 0;
foreach (var pack in query)
{
count++;
var row = pack.row;
foreach (var val in row)
{
Console.Write("{0} ", val);
}
Console.WriteLine();
}
Console.Write("{0} ", count);
Console.WriteLine();
Console.WriteLine("duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
return;
}
//bool run6 = false;
//if (run6)
//{
// tt0 = DateTime.Now;
// foreach (string id in ids)
// {
// var query =
// ts.GetSubjectByObjPred(id, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productFeature")
// .Where(_product => ts.ChkOSubjPredObj(_product, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productFeature", "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature8"))
// .Where(_product => ts.ChkOSubjPredObj(_product, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductType1"))
// .SelectMany(_product => ts.GetDataBySubjPred(_product, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productPropertyNumeric1"))
// ;
// int cnt = query.Count();
// Console.WriteLine(cnt);
// //if (id == "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature19")
// //foreach (var dd in query)
// //{
// // Console.WriteLine("dd={0}", dd);
// //}
// }
// Console.WriteLine("duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
//}
// Контрольный расчет
//{
// string id = "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature19";
// var query =
// ts.GetSubjectByObjPred(id, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productFeature")
// .Where(_product => ts.ChkOSubjPredObj(_product, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productFeature", "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductFeature8"))
// .Where(_product => ts.ChkOSubjPredObj(_product, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/ProductType1"))
// //.SelectMany(_product => ts.GetDataBySubjPred(_product, "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/productPropertyNumeric1"))
// ;
// int cnt = query.Count();
// Console.WriteLine(cnt);
// foreach (var vv in query)
// {
// Console.WriteLine("r=" + vv);
// }
//}
//Console.WriteLine("duration=" + (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
}
public static void Main(string[] args)
{
DateTime tt0 = DateTime.Now;
string path = @"..\..\..\Databases\";
Console.WriteLine("Start RdfTrees");
NameSpaceStore nameSpaceStore = new NameSpaceStore(path);
RdfTrees rtrees = new RdfTrees(path, new StringIntMD5RAMCollision(path), new PredicatesCoding(path), nameSpaceStore, new LiteralStoreSplited(path, nameSpaceStore));
rtrees.LoadTurtle(@"D:\home\FactographDatabases\dataset\dataset1M.ttl", true);
return;
// Разогрев
rtrees.WarmUp();
// Трассировка
XElement tracing = XElement.Load(@"C:\Users\Lena\Downloads\tracing100th.xml");
Console.WriteLine("N_tests = {0}", tracing.Elements().Count());
tt0 = DateTime.Now;
int ecnt = 0, ncnt = 0;
foreach (XElement spo in tracing.Elements())
{
XAttribute s_att = spo.Attribute("subj");
XAttribute p_att = spo.Attribute("pred");
XAttribute o_att = spo.Attribute("obj");
XAttribute r_att = spo.Attribute("res");
string s = s_att == null? null : s_att.Value;
string p = p_att == null ? null : p_att.Value;
string o = o_att == null ? null : o_att.Value;
string res = r_att == null ? null : r_att.Value;
if (spo.Name == "spo_")
{
bool r = rtrees.ChkOSubjPredObj(
s.GetHashCode(),
p.GetHashCode(),
o.GetHashCode());
if ((res == "true" && r) || (res == "false" && !r))
{
ecnt++;
}
else
{
ncnt++;
}
}
else if (spo.Name == "spD_")
{
var lit = rtrees.GetDataBySubjPred(
s.GetHashCode(),
p.GetHashCode()).FirstOrDefault();
if (lit == null)
{
ncnt++;
}
else
{
bool isEq = false;
if (lit.vid == LiteralVidEnumeration.text &&
((Text)lit.Value).Value == res.Substring(1, res.Length - 2))
{
isEq = true;
}
else
{
isEq = lit.ToString() == res;
}
if (isEq)
{
ecnt++;
}
else
{
ncnt++;
}
}
}
else if (spo.Name == "spO")
{
var query = rtrees.GetObjBySubjPred(
s.GetHashCode(),
p.GetHashCode()).OrderBy(v => v).ToArray();
if (query.Count() == 0 && res == "")
{
continue;
}
ecnt++;
}
else if (spo.Name == "Spo_")
{
var query = rtrees.GetSubjectByObjPred(
o.GetHashCode(),
p.GetHashCode()).OrderBy(v => v).ToArray();
if (query.Count() == 0 && res == "")
{
continue;
}
ecnt++;
}
}
// Console.WriteLine("Equal {0} Not equal {1} debug counter {2}", ecnt, ncnt, rtrees.debug_counter);
Console.WriteLine("TOTAL: {0} мс.", (DateTime.Now - tt0).Ticks / 10000L); tt0 = DateTime.Now;
}
