private void Button_Click(object sender, RoutedEventArgs e) { using (SessionNoServer session = new SessionNoServer(systemDir)) { Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); const UInt32 numberOfPersons = 10000; const ushort nodeMaxSize = 5000; const ushort comparisonByteArraySize = sizeof(UInt64); // enough room to hold entire idNumber of a Person const bool comparisonArrayIsCompleteKey = true; const bool addIdCompareIfEqual = false; Person person; session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.None; //mySession.SetTraceAllDbActivity(); BTreeSet <string> stringSet = new BTreeSet <string>(null, session); BTreeSetOidShort <string> stringSetShort = new BTreeSetOidShort <string>(null, session); BTreeMap <string, string> stringMap = new BTreeMap <string, string>(null, session); BTreeMapOidShort <string, string> stringMapShort = new BTreeMapOidShort <string, string>(null, session); CompareByField <Person> compareByField = new CompareByField <Person>("idNumber", session, addIdCompareIfEqual); BTreeSet <Person> bTree = new BTreeSet <Person>(compareByField, session, nodeMaxSize, comparisonByteArraySize, comparisonArrayIsCompleteKey); session.Persist(bTree); // Persist the root of the BTree so that we have something persisted that can be flushed to disk if memory available becomes low for (int i = 0; i < numberOfPersons; i++) { person = new Person(); // session.Persist(person); bTree.AddFast(person); } session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.UseExternalStorageApi = true; session.BeginRead(); BTreeSet <Person> bTree = session.AllObjects <BTreeSet <Person> >().First(); foreach (Person person in (IEnumerable <Person>)bTree) { if (person.IdNumber > 196988888791402) { Console.WriteLine(person); break; } } session.Commit(); } }
private void Button_Click(object sender, RoutedEventArgs e) { using (SessionNoServer session = new SessionNoServer(systemDir)) { Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); const UInt32 numberOfPersons = 10000; const ushort nodeMaxSize = 5000; const ushort comparisonByteArraySize = sizeof(UInt64); // enough room to hold entire idNumber of a Person const bool comparisonArrayIsCompleteKey = true; const bool addIdCompareIfEqual = false; Person person; session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.None; //mySession.SetTraceAllDbActivity(); BTreeSet<string> stringSet = new BTreeSet<string>(null, session); BTreeSetOidShort<string> stringSetShort = new BTreeSetOidShort<string>(null, session); BTreeMap<string, string> stringMap = new BTreeMap<string, string>(null, session); BTreeMapOidShort<string, string> stringMapShort = new BTreeMapOidShort<string, string>(null, session); CompareByField<Person> compareByField = new CompareByField<Person>("idNumber", session, addIdCompareIfEqual); BTreeSet<Person> bTree = new BTreeSet<Person>(compareByField, session, nodeMaxSize, comparisonByteArraySize, comparisonArrayIsCompleteKey); session.Persist(bTree); // Persist the root of the BTree so that we have something persisted that can be flushed to disk if memory available becomes low for (int i = 0; i < numberOfPersons; i++) { person = new Person(); // session.Persist(person); bTree.AddFast(person); } session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.UseExternalStorageApi = true; session.BeginRead(); BTreeSet<Person> bTree = session.AllObjects<BTreeSet<Person>>().First(); foreach (Person person in (IEnumerable<Person>)bTree) { if (person.IdNumber > 196988888791402) { Console.WriteLine(person); break; } } session.Commit(); } }
public void ingestData() { if (Directory.Exists(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir))) Directory.Delete(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir), true); // remove systemDir from prior runs and all its databases. Directory.CreateDirectory(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir)); File.Copy(s_licenseDbFile, Path.Combine(SessionBase.BaseDatabasePath, s_systemDir, "4.odb")); using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, true)) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; Graph g = new Graph(session); // SCHEMA VertexType userType = g.NewVertexType("User"); EdgeType friendEdgeType = g.NewEdgeType("Friend", true, userType, userType); PropertyType countryProperty = userType.NewProperty("country", DataType.String, PropertyKind.NotIndexed); PropertyType incomeProperty = userType.NewProperty("income", DataType.Long, PropertyKind.NotIndexed); PropertyType friendshipStartProperty = friendEdgeType.NewProperty("start", DataType.DateTime, PropertyKind.NotIndexed); // DATA int lineNumber = 0; long fiendsCt = 0; int stop = (int)Math.Pow(2, 26); // make sure to create enough of these for (int i = 1; i < stop; i++) g.NewVertex(userType); session.Commit(); session.BeginUpdate(); foreach (string line in File.ReadLines(s_inputData)) { if (++lineNumber % 10000 == 0) Console.WriteLine("Parsing user " + lineNumber + ", friends total: " + fiendsCt + " at " + DateTime.Now); string[] fields = line.Split(' '); Vertex aUser = null; foreach (string s in fields) { if (s.Length > 0) { if (aUser == null) aUser = new Vertex(g, userType, int.Parse(s)); else { ++fiendsCt; Vertex aFriend = new Vertex(g, userType, int.Parse(s)); if (fiendsCt % 2 == 0) aFriend.SetProperty(countryProperty, "Sweden"); // just some random stuff else aFriend.SetProperty(incomeProperty, fiendsCt); // just some random stuff Edge edge = friendEdgeType.NewEdge(aUser, aFriend); if (fiendsCt % 2 == 0) edge.SetProperty(friendshipStartProperty, DateTime.Now); } } } if (DataCache.MaximumMemoryUse <= 27000000000) { if (lineNumber >= 20000) // remove this condition if you have time to wait a long while... break; } } Console.WriteLine("Done importing " + lineNumber + " users with " + fiendsCt + " friends"); session.Commit(); } }
static void Main(string[] args) { bool import = args.Length > 0 && args[0].ToLower() == "-import"; bool dirExist = Directory.Exists(s_systemDir); SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false; if (import || !dirExist) { if (dirExist) { Directory.Delete(s_systemDir, true); // remove systemDir from prior runs and all its databases. } Directory.CreateDirectory(s_systemDir); using (SessionNoServer session = new SessionNoServer(s_systemDir)) { DataCache.MaximumMemoryUse = 12000000000; // 12 GB, set this to what fits your case SessionBase.BTreeAddFastTransientBatchSize = 10; // reduces memory usage Vertex[] ratingVertices = new Vertex[10]; session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; Graph g = new Graph(session); // SCHEMA VertexType userType = g.NewVertexType("User"); PropertyType genderType = userType.NewProperty("Gender", DataType.Integer, PropertyKind.Indexed); VertexType ratingType = g.NewVertexType("Rating"); PropertyType ratingValuePropertyType = ratingType.NewProperty("RatingValue", DataType.Integer, PropertyKind.Indexed); EdgeType ratingEdgeType = g.NewEdgeType("UserToRating", true, userType, ratingType); EdgeType ratingOfType = g.NewEdgeType("RatingOf", false, userType, userType); PropertyType ratingEdgePropertyType = ratingOfType.NewProperty("Rating", DataType.Integer, PropertyKind.Indexed); // DATA using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "gender.dat"))) { using (StreamReader file = new System.IO.StreamReader(stream)) { string line; int lineNumber = 0; while ((line = file.ReadLine()) != null) { lineNumber++; string[] fields = line.Split(','); Vertex aUser = userType.NewVertex(); aUser.SetProperty(genderType, (int)fields[1][0] == 'M' ? Gender.Male : fields[1][0] == 'F' ? Gender.Female : Gender.Unknown); } Console.WriteLine("Done importing " + lineNumber + " users"); } } using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "ratings.dat"))) { using (StreamReader file = new System.IO.StreamReader(stream)) { string line; int lineNumber = 0; Vertex rater = null; int raterId; int priorRaterId = -1; while ((line = file.ReadLine()) != null) { lineNumber++; if (lineNumber % 1000000 == 0) { Console.WriteLine("Parsing rating # " + lineNumber); } string[] fields = line.Split(','); raterId = int.Parse(fields[0]); if (raterId != priorRaterId) { rater = userType.GetVertex(raterId); } priorRaterId = raterId; int ratedId = int.Parse(fields[1]); int rating = int.Parse(fields[2]); Vertex ratingVertex = ratingVertices[rating - 1]; if (ratingVertex == null) { ratingVertex = ratingType.NewVertex(); ratingVertex.SetProperty(ratingValuePropertyType, rating); ratingVertices[rating - 1] = ratingVertex; } Vertex rated = userType.GetVertex(ratedId); Edge aRatingOf = ratingOfType.NewEdge(rater, rated); aRatingOf.SetProperty(ratingEdgePropertyType, rating); Edge userRating = ratingEdgeType.NewEdge(rated, ratingVertex); } Console.WriteLine("Done importing " + lineNumber + " ratings"); } } session.Commit(); } } // Query using (SessionNoServer session = new SessionNoServer(s_systemDir)) { session.BeginRead(); Graph g = Graph.Open(session); VertexType userType = g.FindVertexType("User"); PropertyType genderType = userType.FindProperty("Gender"); VertexType ratingType = g.FindVertexType("Rating"); PropertyType ratingValuePropertyType = ratingType.FindProperty("RatingValue"); EdgeType ratingEdgeType = g.FindEdgeType("UserToRating"); EdgeType ratingOfType = g.FindEdgeType("RatingOf"); PropertyType ratingEdgePropertyType = ratingOfType.FindProperty("Rating"); // Complex queries int ct = 0; // Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated Vertex someUser = userType.GetVertex(1); var similarRatings = (from Edge e in someUser.GetEdges(ratingOfType, Direction.Out) from Edge edge in e.Head.GetEdges(ratingOfType, Direction.Out) where someUser != edge.Head where ((int)e.GetProperty(ratingEdgePropertyType) == (int)edge.GetProperty(ratingEdgePropertyType)) select edge.Tail).Distinct(); var someUserRated = from Edge e in someUser.GetEdges(ratingOfType, Direction.Out) select e.Head; var recommendedProfles = from v in similarRatings from Edge e in v.GetEdges(ratingOfType, Direction.Out) where someUserRated.Contains(e.Head) == false select e.Head; Console.WriteLine("Some user's rated profiles"); ct = 0; foreach (Vertex v in someUserRated) { if (ct++ % 50 == 0) // don't print them all ! { Console.WriteLine("User id: " + v.VertexId); } } Console.WriteLine("Number of some user's rated profiles: " + ct); Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles"); ct = 0; foreach (Vertex v in similarRatings) { if (ct++ % 50 == 0) // don't print them all ! { Console.WriteLine("User id: " + v.VertexId); } } Console.WriteLine("Number of matching profiles: " + ct); Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated"); ct = 0; foreach (Vertex v in recommendedProfles) { if (ct++ % 5000 == 0) // don't print them all ! { Console.WriteLine("User id: " + v.VertexId); } } Console.WriteLine("Number of matching profiles: " + ct); // Get all female users with less than 50 ratings Console.WriteLine(); var females = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Female select u; var femalesLess50Ratings = from f in females where f.GetNumberOfEdges(ratingEdgeType, Direction.Out) < 50 select f; Console.WriteLine("Female users with less than 50 ratings"); ct = 0; foreach (Vertex f in femalesLess50Ratings) { long count = f.GetNumberOfEdges(ratingEdgeType, Direction.Out); if (ct++ % 5000 == 0) // don't print them all ! { Console.WriteLine("User id: " + f.VertexId + "\tnumber of ratings: " + count); } } Console.WriteLine("Number of females with fewer than 50 ratings: " + ct); Console.WriteLine(); // Get all male users with at least one 10 rating Console.WriteLine(); var rated10vertex = (from v in ratingType.GetVertices() where ((int)v.GetProperty(ratingValuePropertyType)) == 10 select v).First(); var rated10 = (from e in rated10vertex.GetEdges(ratingEdgeType, Direction.In) select e.GetVertex(Direction.Out)).Distinct(); var rated10male = from Vertex v in rated10 where ((Gender)v.GetProperty(genderType)) == Gender.Male select v; Console.WriteLine("Males with at least one 10 rating"); ct = 0; foreach (Vertex v in rated10male) { if (ct++ % 5000 == 0) // don't print them all ! { Console.WriteLine("User id: " + v.VertexId); } } Console.WriteLine("Number of males with at least one 10 rating: " + ct); Console.WriteLine(); // Get the first 10 male users who have rated at least 3 of the same profiles as the given user. Console.WriteLine("10 male users who have rated at least 3 of the same profiles as the given user"); var males = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Male select u; var someUserHasRated = from Edge o in someUser.GetEdges(ratingOfType, Direction.Out) select o.Head; var first10withSame3ratedAs = from m in males where (from Edge r in m.GetEdges(ratingOfType, Direction.Out) select r.Head).Intersect(someUserHasRated).ToArray().Length >= 3 select m; ct = 0; foreach (Vertex v in first10withSame3ratedAs) { if (++ct > 10) { break; } Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine(); // Statistical queries // Get the 20 profiles with the most ratings var top20mostRatings = (from v in userType.GetVertices() orderby v.GetNumberOfEdges(ratingEdgeType, Direction.Out) descending select v).Take(20); Console.WriteLine("20 profiles with the most ratings"); ct = 0; foreach (Vertex v in top20mostRatings) { int count = (int)v.GetNumberOfEdges(ratingEdgeType, Direction.Out); Console.WriteLine("User id: " + v.VertexId + "\tnumber of ratings: " + count); } Console.WriteLine(); var ratingsVertexEnum = from v in ratingType.GetVertices() orderby v.GetProperty(ratingValuePropertyType) descending select v; Vertex rating10Vertex = ratingsVertexEnum.First(); // Get the 20 best rated profiles regardless of gender var top = from u in userType.GetVertices() let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out) orderby edgeCt descending select new { u, edgeCt }; Console.WriteLine("20 best rated profiles regardless of gender"); ct = 0; foreach (var v in top) { if (++ct > 20) { break; } Console.WriteLine("User id: " + v.u.VertexId + "\t10 ratings: " + v.edgeCt); } Console.WriteLine(); // Get the 20 best rated males Console.WriteLine("20 best rated male profiles"); var top20males = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Male let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out) orderby edgeCt descending select new { u, edgeCt }; ct = 0; foreach (var v in top20males) { if (++ct > 20) { break; } Console.WriteLine("Male User id: " + v.u.VertexId + " \t10 ratings: " + v.edgeCt); } Console.WriteLine(); // Get the 20 best rated females Console.WriteLine("20 best rated female profiles"); var top20females = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Female let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out) orderby edgeCt descending select new { u, edgeCt }; ct = 0; foreach (var v in top20females) { if (++ct > 20) { break; } Console.WriteLine("Female User id: " + v.u.VertexId + "\t10 ratings: " + v.edgeCt); } session.Commit(); } }
public void AddVertices() { using (var session = new SessionNoServer(@"d:\graphtest2")) { session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; DataCache.MaximumMemoryUse = 4000000000; var sw = new Stopwatch(); sw.Start(); //int i = 0; //var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0, // 3, session); //DatabaseLocation bl = session.NewLocation(dbl); //session.Commit(false); // Assert.That(bl!=null); // var db = session.OpenDatabase(15, true); var graph = new Graph(session); session.BeginUpdate(); session.Persist(graph); //define schema Trace.Wri VertexType concept = graph.NewVertexType("Concept"); PropertyType conceptName = concept.NewProperty("ConceptName", DataType.String, PropertyKind.Unique); PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed); PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed); VertexType instance = graph.NewVertexType("Instance", concept); PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceName = instance.NewProperty("InstanceName", DataType.String, PropertyKind.Unique); PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency", DataType.Integer, PropertyKind.NotIndexed); //VertexType attributes = graph.NewVertexType("Attribute"); ////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); //EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); //PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, // PropertyKind.NotIndexed); //VertexType synonym = graph.NewVertexType("Synonym", namedScore); //PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); //EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance); PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed); PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed); PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed); EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept); //LRVertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //LRVertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //LREdge edge = graph.NewEdge(cooccurence, vx1, vx2); Vertex v2 = graph.NewVertex(concept); v2.SetProperty(conceptName, "factor"); Vertex v3 = graph.NewVertex(instance); v3.SetProperty(instanceName, "age"); session.Commit(); } using (var session = new SessionNoServer(@"d:\graphtest2")) { //session.DefaultDatabaseLocation().CompressPages = true; DataCache.MaximumMemoryUse = 4000000000; var sw = new Stopwatch(); sw.Start(); //int i = 0; session.BeginRead(); var graph = Graph.Open(session); //define schema Trace.Wri VertexType[] vertexTypes = graph.FindVertexTypes(); //vertexTypes.Select(x => x.TypeName).PrintDump(); EdgeType[] edgeTypes = graph.FindEdgeTypes(); VertexType concept = vertexTypes.FirstOrDefault(x => x.TypeName == "Concept") ?? graph.NewVertexType("Concept"); PropertyType conceptName = concept.FindProperty("ConceptName"); Assert.IsNotNull(conceptName, "ConceptName"); PropertyType conceptSize = concept.FindProperty("ConceptSize"); Assert.IsNotNull(conceptSize, "ConceptSize"); PropertyType conceptFrequency = concept.FindProperty("ConceptFrequency"); //PropertyType similarity = concept.NewProperty("Similarity", VelocityGraph.DataType.Double, // VelocityGraph.PropertyKind.NotIndexed); PropertyType vagueness = concept.FindProperty("Vagueness");//, DataType.Double,PropertyKind.NotIndexed); VertexType instance = vertexTypes.FirstOrDefault(x => x.TypeName == "Instance"); PropertyType instanceSize = instance.FindProperty("InstanceSize"); PropertyType instanceName = instance.FindProperty("InstanceName"); PropertyType instanceFrequency = instance.FindProperty("InstanceFrequency"); //VertexType attributes = graph.NewVertexType("Attribute"); ////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); //EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); //PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, // PropertyKind.NotIndexed); //VertexType synonym = graph.NewVertexType("Synonym", namedScore); //PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); //EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = edgeTypes.FirstOrDefault(x => x.TypeName == "IsA"); PropertyType frequency = isA.FindProperty("frequency"); PropertyType popularity = isA.FindProperty("popularity"); PropertyType ZipfSlope = isA.FindProperty("zipf_slope"); PropertyType ZipfPearson = isA.FindProperty("zipf_pearson"); EdgeType cooccurence = graph.FindEdgeType("Coocurrence"); //LRVertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //LRVertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //LREdge edge = graph.NewEdge(cooccurence, vx1, vx2); Assert.IsNotNull(conceptName); Vertex f = graph.FindVertex(conceptName, "factor"); var inst = graph.FindVertex(instanceName, "age"); Assert.IsNotNull(f); Assert.IsNotNull(inst); } //if (instanceVertex == null) }
// [Test] public void TestVelecoityBuildLocal() { using (var session = new SessionNoServer(@"d:\graphtest")) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; DataCache.MaximumMemoryUse = 2000000000; //var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0, // 3, session); //DatabaseLocation bl = session.NewLocation(dbl); //session.Commit(false); // Assert.That(bl!=null); var graph = new Graph(session); session.Persist(graph); //define schema // session.BeginUpdate(); VertexType namedScore = graph.NewVertexType("NamedScore"); PropertyType name = namedScore.NewProperty("Name", DataType.String, PropertyKind.Indexed); PropertyType int_score = namedScore.NewProperty("IntScore", DataType.Integer, PropertyKind.NotIndexed); PropertyType double_score = namedScore.NewProperty("DoubleScore", DataType.Double, PropertyKind.NotIndexed); VertexType concept = graph.NewVertexType("Concept"); PropertyType conceptName = concept.NewProperty("Name", DataType.String, PropertyKind.Unique); PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed); PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed); VertexType instance = graph.NewVertexType("Instance", concept); PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceName = instance.NewProperty("Name", DataType.String, PropertyKind.Unique); VertexType attributes = graph.NewVertexType("Attribute", namedScore); //EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, PropertyKind.NotIndexed); VertexType synonym = graph.NewVertexType("Synonym", namedScore); PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance); PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed); PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed); PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed); EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept); //Vertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //Vertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //Edge edge = graph.NewEdge(cooccurence, vx1, vx2); using (var llz = new StreamReader(@"d:\isa_core.txt")) { string lastConcept = string.Empty; while (!llz.EndOfStream) { string ln = llz.ReadLine(); if (string.IsNullOrEmpty(ln)) { continue; } string[] items = ln.Split(new[] { '\t' }); if (items.Length < 4) { continue; } int id = -1; if (!int.TryParse(items[2], out id)) { continue; } var conceptVertex = graph.FindVertex(conceptName, items[0], false); if (conceptVertex == null) { conceptVertex = graph.NewVertex(concept); conceptVertex.SetProperty(conceptName, items[0]); conceptVertex.SetProperty(conceptFrequency, int.Parse(items[4])); conceptVertex.SetProperty(conceptSize, int.Parse(items[5])); double d = double.NaN; double.TryParse(items[6], out d); conceptVertex.SetProperty(vagueness, d); d = double.NaN; double.TryParse(items[7], out d); conceptVertex.SetProperty(ZipfSlope, d); d = double.NaN; double.TryParse(items[8], out d); conceptVertex.SetProperty(ZipfPearson, d); } var instanceVertex = graph.FindVertex(instanceName, items[1], false); if (instanceVertex == null) { instanceVertex = graph.NewVertex(instance); instanceVertex.SetProperty(instanceFrequency, int.Parse(items[9])); instanceVertex.SetProperty(instanceSize, int.Parse(items[10])); } var isaedge = graph.NewEdge(isA, conceptVertex, instanceVertex); isaedge.SetProperty(frequency, int.Parse(items[2])); isaedge.SetProperty(popularity, int.Parse(items[3])); } session.Commit(); } } }
public void Create1Vertices(bool vertexIdSetPerVertexType) { DataCache.MaximumMemoryUse = 10000000000; // 10 GB bool dirExist = Directory.Exists(systemDir); try { if (Directory.Exists(systemDir)) { Directory.Delete(systemDir, true); // remove systemDir from prior runs and all its databases. } Directory.CreateDirectory(systemDir); File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); } catch { File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); } using (SessionNoServer session = new SessionNoServer(systemDir, 5000, false, true)) { session.BeginUpdate(); Graph g = new Graph(session, vertexIdSetPerVertexType); session.Persist(g); VertexType userType = g.NewVertexType("User"); VertexType otherType = g.NewVertexType("Other"); PropertyType userNamePropertyType = g.NewVertexProperty(userType, "NAME", DataType.String, PropertyKind.Indexed); VertexType powerUserType = g.NewVertexType("PowerUser", userType); EdgeType userFriendEdgeType = g.NewEdgeType("Friend", true, userType, userType); EdgeType userBestFriendEdgeType = g.NewEdgeType("Best Friend", true, userType, userType, userFriendEdgeType); EdgeType otherEdgeType = g.NewEdgeType("Other", true, userType, userType); PropertyType bestFriendPropertyType = g.NewEdgeProperty(userFriendEdgeType, "START", DataType.DateTime, PropertyKind.Indexed); Vertex kinga = userType.NewVertex(); Vertex robin = userType.NewVertex(); Vertex mats = powerUserType.NewVertex(); Vertex chiran = powerUserType.NewVertex(); Vertex other = otherType.NewVertex(); Edge bestFriend = kinga.AddEdge(userBestFriendEdgeType, robin); Edge otherEdge = kinga.AddEdge(otherEdgeType, robin); DateTime now = DateTime.Now; mats.SetProperty("Address", 1); bestFriend.SetProperty(bestFriendPropertyType, now); kinga.SetProperty(userNamePropertyType, "Kinga"); if (g.VertexIdSetPerType == false) { mats.SetProperty(userNamePropertyType, "Mats"); } else { try { mats.SetProperty(userNamePropertyType, "Mats"); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } } try { other.SetProperty(userNamePropertyType, "Mats"); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } try { otherEdge.SetProperty(bestFriendPropertyType, now); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } Vertex findMats = userNamePropertyType.GetPropertyVertex("Mats", true); var list = userNamePropertyType.GetPropertyVertices("Mats", true).ToList(); //Edge findWhen = bestFriendPropertyType.GetPropertyEdge(now); //var list2 = bestFriendPropertyType.GetPropertyEdges(now); Console.WriteLine(findMats); // session.Commit(); // session.BeginRead(); PropertyType adressProperty = g.FindVertexProperty(powerUserType, "Address"); Vertex find1 = adressProperty.GetPropertyVertex(1, true); session.Abort(); /*session.BeginUpdate(); * g.Unpersist(session); * session.Commit(); * dirExist = Directory.Exists(systemDir); * try * { * if (Directory.Exists(systemDir)) * Directory.Delete(systemDir, true); // remove systemDir from prior runs and all its databases. * Directory.CreateDirectory(systemDir); * File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); * } * catch * { * File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); * }*/ } using (SessionNoServer session = new SessionNoServer(systemDir, 5000, false, true)) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.None; Graph g = new Graph(session, vertexIdSetPerVertexType); session.Persist(g); Graph g2 = new Graph(session); session.Persist(g2); Graph g3 = new Graph(session); session.Persist(g3); UInt32 dbNum = session.DatabaseNumberOf(typeof(Graph)); Graph g4 = (Graph)session.Open(dbNum, 2, 1, true); // g4 == g Graph g5 = (Graph)session.Open(dbNum, 2, 2, true); // g5 == g2 Graph g6 = (Graph)session.Open(dbNum, 2, 3, true); // g6 == g3 for (int i = 4; i < 8; i++) { Graph gt = new Graph(session); session.Persist(gt); } Graph g7 = new Graph(session); Placement place = new Placement(dbNum, 15); session.Persist(place, g7); // SCHEMA VertexType userType = g.NewVertexType("User"); VertexType locationType = g.NewVertexType("Location"); VertexType aVertexType = g.NewVertexType("A"); VertexType bVertexType = g.NewVertexType("B"); VertexType cVertexType = g.NewVertexType("C"); EdgeType uEdge = g.NewEdgeType("unrestricted", true); Vertex aVertex = g.NewVertex(aVertexType); Vertex bVertex = g.NewVertex(bVertexType); Vertex cVertex = g.NewVertex(cVertexType); Edge abEdge = (Edge)aVertex.AddEdge("unrestricted", bVertex); Edge bcEdge = (Edge)aVertex.AddEdge("unrestricted", cVertex); Dictionary <Vertex, HashSet <Edge> > traverse = aVertex.Traverse(uEdge, Direction.Out); abEdge.Remove(); Dictionary <Vertex, HashSet <Edge> > traverse2 = aVertex.Traverse(uEdge, Direction.Out); EdgeType friendEdgeType = g.NewEdgeType("Friend", true, userType, userType); EdgeType userLocationEdgeType = g.NewEdgeType("UserLocation", true, userType, locationType); // DATA Random rand = new Random(5); for (int i = 0; i < numberOfUserVertices / 100; i++) { int vId = rand.Next(numberOfUserVertices); try { if (g.VertexIdSetPerType) { userType.GetVertex(vId); } else { g.GetVertex(vId); } try { userType.NewVertex(vId); Assert.Fail(); } catch (VertexAllreadyExistException) { } } catch (VertexDoesNotExistException) { userType.NewVertex(vId); userType.GetVertex(vId); } } for (int i = 0; i < numberOfUserVertices / 10000; i++) { int vId = rand.Next(numberOfUserVertices); try { Vertex v = userType.GetVertex(vId); v.SetProperty("test", 1); } catch (VertexDoesNotExistException) { } } for (int i = 0; i < numberOfUserVertices / 10000; i++) { int vId = rand.Next(numberOfUserVertices); try { Vertex v = userType.GetVertex(vId); userType.RemoveVertex(v); } catch (VertexDoesNotExistException) { } } foreach (Vertex v in userType.GetVertices().ToArray()) { userType.RemoveVertex(v); } Assert.AreEqual(0, userType.GetVertices().Count()); for (int i = 100000; i < numberOfUserVertices; i++) { userType.NewVertex(); } for (int i = 1; i < 100000; i++) { userType.NewVertex(); } for (int i = 1; i < numberOfLocationVertices; i++) { locationType.NewVertex(); } session.Commit(); session.BeginRead(); foreach (var x in session.AllObjects <BTreeSet <Range <VertexId> > >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeSet <EdgeType> >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeSet <EdgeIdVertexId> >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <EdgeId, VelocityDbList <ElementId> > >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <string, PropertyType> >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <string, EdgeType> >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <string, VertexType> >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <VertexId, BTreeSet <EdgeIdVertexId> > >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <VertexType, BTreeMap <VertexId, BTreeSet <EdgeIdVertexId> > > >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } foreach (var x in session.AllObjects <BTreeMap <EdgeType, BTreeMap <VertexType, BTreeMap <VertexId, BTreeSet <EdgeIdVertexId> > > > >(false, true)) { Assert.True(x.ToDoBatchAddCount == 0); } session.Commit(); Validate(); } }
static void Main(string[] args) { bool import = args.Length > 0 && args[0].ToLower() == "-import"; bool dirExist = Directory.Exists(s_systemDir); if (import || !dirExist) { if (dirExist) Directory.Delete(s_systemDir, true); // remove systemDir from prior runs and all its databases. Directory.CreateDirectory(s_systemDir); File.Copy(s_licenseDbFile, Path.Combine(s_systemDir, "4.odb")); using (SessionNoServer session = new SessionNoServer(s_systemDir)) { DataCache.MaximumMemoryUse = 10000000000; // 10 GB, set this to what fits your case session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; Graph g = new Graph(session); session.Persist(g); // SCHEMA VertexType userType = g.NewVertexType("User"); PropertyType genderType = userType.NewProperty("Gender", DataType.Integer, PropertyKind.Indexed); VertexType ratingType = g.NewVertexType("Rating"); PropertyType ratingValuePropertyType = ratingType.NewProperty("RatingValue", DataType.Integer, PropertyKind.Indexed); EdgeType ratingEdgeType = g.NewEdgeType("UserToRating", true, userType, ratingType); EdgeType ratingOfType = g.NewEdgeType("RatingOf", false, userType, userType); PropertyType ratingEdgePropertyType = ratingOfType.NewProperty("Rating", DataType.Integer, PropertyKind.Indexed); // DATA using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "gender.dat"))) { using (StreamReader file = new System.IO.StreamReader(stream)) { string line; int lineNumber = 0; while ((line = file.ReadLine()) != null) { lineNumber++; string[] fields = line.Split(','); Vertex aUser = userType.NewVertex(); aUser.SetProperty(genderType, (int)fields[1][0] == 'M' ? Gender.Male : fields[1][0] == 'F' ? Gender.Female : Gender.Unknown); } Console.WriteLine("Done importing " + lineNumber + " users"); } } using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "ratings.dat"))) { using (StreamReader file = new System.IO.StreamReader(stream)) { string line; int lineNumber = 0; Vertex rater = null; int raterId; int priorRaterId = -1; while ((line = file.ReadLine()) != null) { lineNumber++; if (lineNumber % 1000000 == 0) Console.WriteLine("Parsing rating # " + lineNumber); string[] fields = line.Split(','); raterId = int.Parse(fields[0]); if (raterId != priorRaterId) rater = userType.GetVertex(raterId); priorRaterId = raterId; int ratedId = int.Parse(fields[1]); int rating = int.Parse(fields[2]); Vertex ratingVertex = (from v in ratingType.GetVertices() where ((int)v.GetProperty(ratingValuePropertyType)) == rating select v).FirstOrDefault(); if (ratingVertex == null) { ratingVertex = ratingType.NewVertex(); ratingVertex.SetProperty(ratingValuePropertyType, rating); } Vertex rated = userType.GetVertex(ratedId); Edge aRatingOf = ratingOfType.NewEdge(rater, rated); aRatingOf.SetProperty(ratingEdgePropertyType, rating); Edge userRating = ratingEdgeType.NewEdge(rated, ratingVertex); if (lineNumber >= 10000000) // remove this condition if you have time to wait a while and you have at least 16GB of RAM memory break; } Console.WriteLine("Done importing " + lineNumber + " ratings"); } } session.Commit(); } } // Query using (SessionNoServer session = new SessionNoServer(s_systemDir)) { session.BeginRead(); Graph g = Graph.Open(session); VertexType userType = g.FindVertexType("User"); PropertyType genderType = userType.FindProperty("Gender"); VertexType ratingType = g.FindVertexType("Rating"); PropertyType ratingValuePropertyType = ratingType.FindProperty("RatingValue"); EdgeType ratingEdgeType = g.FindEdgeType("UserToRating"); PropertyType userRatingEdgePropertyType = ratingEdgeType.FindProperty("UserRating"); EdgeType ratingOfType = g.FindEdgeType("RatingOf"); PropertyType ratingEdgePropertyType = ratingOfType.FindProperty("Rating"); // Complex queries var ratingsVertexEnum = from v in ratingType.GetVertices() orderby v.GetProperty(ratingValuePropertyType) descending select v; int ct = 0; // Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated Vertex someUser = userType.GetVertex(1); var similarRatings = (from Edge e in someUser.GetEdges(ratingOfType, Direction.Out) from Edge edge in e.Head.GetEdges(ratingOfType, Direction.Out) where someUser != edge.Head where ((int)e.GetProperty(ratingEdgePropertyType) == (int)edge.GetProperty(ratingEdgePropertyType)) select edge.Tail).Distinct(); var someUserRated = from Edge e in someUser.GetEdges(ratingOfType, Direction.Out) select e.Head; var recommendedProfles = from v in similarRatings from Edge e in v.GetEdges(ratingOfType, Direction.Out) where someUserRated.Contains(e.Head) == false select e.Head; Console.WriteLine("Some user's rated profiles"); ct = 0; foreach (Vertex v in someUserRated) { ct++; //Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine("Number of some user's rated profiles: " + ct); Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles"); ct = 0; foreach (Vertex v in similarRatings) { ct++; //Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine("Number of matching profiles: " + ct); Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated"); ct = 0; foreach (Vertex v in recommendedProfles) { if (ct++ % 5000 == 0) // don't print them all ! Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine("Number of matching profiles: " + ct); // Get all female users with less than 50 ratings Console.WriteLine(); var females = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Female select u; var femalesLess50Ratings = from f in females where f.GetNumberOfEdges(ratingEdgeType, Direction.Out) < 50 select f; Console.WriteLine("Female users with less than 50 ratings"); ct = 0; foreach (Vertex f in femalesLess50Ratings) { long count = f.GetNumberOfEdges(ratingEdgeType, Direction.Out); if (ct++ % 5000 == 0) // don't print them all ! Console.WriteLine("User id: " + f.VertexId + "\tnumber of ratings: " + count); } Console.WriteLine("Number of females with fewer than 50 ratings: " + ct); Console.WriteLine(); // Get all male users with at least one 10 rating Console.WriteLine(); var rated10vertex = (from v in ratingType.GetVertices() where ((int)v.GetProperty(ratingValuePropertyType)) == 10 select v).First(); var rated10 = (from e in rated10vertex.GetEdges(ratingEdgeType, Direction.In) select e.GetVertex(Direction.Out)).Distinct(); var rated10male = from Vertex v in rated10 where ((Gender)v.GetProperty(genderType)) == Gender.Male select v; Console.WriteLine("Males with at least one 10 rating"); ct = 0; foreach (Vertex v in rated10male) { if (ct++ % 5000 == 0) // don't print them all ! Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine("Number of males with at least one 10 rating: " + ct); Console.WriteLine(); // Get the first 10 male users who have rated at least 3 of the same profiles as the given user. Console.WriteLine("10 male users who have rated at least 3 of the same profiles as the given user"); var males = from u in userType.GetVertices() where ((Gender)u.GetProperty(genderType)) == Gender.Male select u; var someUserHasRated = from Edge o in someUser.GetEdges(ratingOfType, Direction.Out) select o.Head; var first10withSame3ratedAs = from m in males where (from Edge r in m.GetEdges(ratingOfType, Direction.Out) select r.Head).Intersect(someUserHasRated).ToArray().Length >= 3 select m; ct = 0; foreach (Vertex v in first10withSame3ratedAs) { if (++ct > 10) break; Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine(); // Statistical queries // Get the 20 profiles with the most ratings var top20mostRatings = (from v in userType.GetVertices() orderby v.GetNumberOfEdges(ratingEdgeType, Direction.Out) descending select v).Take(20); Console.WriteLine("20 profiles with the most ratings"); ct = 0; foreach (Vertex v in top20mostRatings) { int count = (int)v.GetNumberOfEdges(ratingEdgeType, Direction.Out); Console.WriteLine("User id: " + v.VertexId + "\tnumber of ratings: " + count); } Console.WriteLine(); // Get the 20 best rated profiles regardless of gender var top = from v in ratingsVertexEnum from e in v.GetEdges(ratingEdgeType, Direction.In) select e.GetVertex(Direction.Out); Console.WriteLine("20 best rated profiles regardless of gender"); ct = 0; foreach (Vertex v in top) { if (++ct > 20) break; Console.WriteLine("User id: " + v.VertexId); } Console.WriteLine(); // Get the 20 best rated males Console.WriteLine("20 best rated male profiles"); var top20males = from Vertex v in top where ((Gender)v.GetProperty(genderType)) == Gender.Male select v; ct = 0; foreach (Vertex v in top20males) { if (++ct > 20) break; Console.WriteLine("Male User id: " + v.VertexId); } Console.WriteLine(); // Get the 20 best rated females Console.WriteLine("20 best rated female profiles"); var top20females = from Vertex v in top where ((Gender)v.GetProperty(genderType)) == Gender.Female select v; ct = 0; foreach (Vertex v in top20females) { if (++ct > 20) break; Console.WriteLine("Female User id: " + v.VertexId); } session.Commit(); } }
public void ingestData() { if (Directory.Exists(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir))) { Directory.Delete(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir), true); // remove systemDir from prior runs and all its databases. } Directory.CreateDirectory(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir)); using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; Graph g = new Graph(session); // SCHEMA VertexType userType = g.NewVertexType("User"); EdgeType friendEdgeType = g.NewEdgeType("Friend", true, userType, userType); PropertyType countryProperty = userType.NewProperty("country", DataType.String, PropertyKind.NotIndexed); PropertyType incomeProperty = userType.NewProperty("income", DataType.Long, PropertyKind.NotIndexed); PropertyType friendshipStartProperty = friendEdgeType.NewProperty("start", DataType.DateTime, PropertyKind.NotIndexed); // DATA int lineNumber = 0; long fiendsCt = 0; int stop = (int)Math.Pow(2, 26); // make sure to create enough of these for (int i = 1; i < stop; i++) { g.NewVertex(userType); } session.Commit(); session.BeginUpdate(); foreach (string line in File.ReadLines(s_inputData)) { if (++lineNumber % 10000 == 0) { Console.WriteLine("Parsing user " + lineNumber + ", friends total: " + fiendsCt + " at " + DateTime.Now); } string[] fields = line.Split(' '); Vertex aUser = null; foreach (string s in fields) { if (s.Length > 0) { if (aUser == null) { aUser = new Vertex(g, userType, int.Parse(s)); } else { ++fiendsCt; Vertex aFriend = new Vertex(g, userType, int.Parse(s)); if (fiendsCt % 2 == 0) { aFriend.SetProperty(countryProperty, "Sweden"); // just some random stuff } else { aFriend.SetProperty(incomeProperty, fiendsCt); // just some random stuff } Edge edge = friendEdgeType.NewEdge(aUser, aFriend); if (fiendsCt % 2 == 0) { edge.SetProperty(friendshipStartProperty, DateTime.Now); } } } } if (DataCache.MaximumMemoryUse <= 27000000000) { if (lineNumber >= 100000) // remove this condition if you have time to wait a long while... { break; } } } Console.WriteLine("Done importing " + lineNumber + " users with " + fiendsCt + " friends"); session.Commit(); } }
public void AddVertices() { using (var session = new SessionNoServer(@"d:\graphtest2")) { session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; DataCache.MaximumMemoryUse = 4000000000; var sw = new Stopwatch(); sw.Start(); //int i = 0; //var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0, // 3, session); //DatabaseLocation bl = session.NewLocation(dbl); //session.Commit(false); // Assert.That(bl!=null); // var db = session.OpenDatabase(15, true); session.BeginUpdate(); var graph = new Graph(session); //define schema Trace.Wri VertexType concept = graph.NewVertexType("Concept"); PropertyType conceptName = concept.NewProperty("ConceptName", DataType.String, PropertyKind.Unique); PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed); PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed); VertexType instance = graph.NewVertexType("Instance", concept); PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceName = instance.NewProperty("InstanceName", DataType.String, PropertyKind.Unique); PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency", DataType.Integer, PropertyKind.NotIndexed); //VertexType attributes = graph.NewVertexType("Attribute"); ////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); //EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); //PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, // PropertyKind.NotIndexed); //VertexType synonym = graph.NewVertexType("Synonym", namedScore); //PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); //EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance); PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed); PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed); PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed); EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept); //LRVertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //LRVertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //LREdge edge = graph.NewEdge(cooccurence, vx1, vx2); Vertex v2 = graph.NewVertex(concept); v2.SetProperty(conceptName, "factor"); Vertex v3 = graph.NewVertex(instance); v3.SetProperty(instanceName, "age"); session.Commit(); } using (var session = new SessionNoServer(@"d:\graphtest2")) { //session.DefaultDatabaseLocation().CompressPages = true; DataCache.MaximumMemoryUse = 4000000000; var sw = new Stopwatch(); sw.Start(); //int i = 0; session.BeginRead(); var graph = Graph.Open(session); //define schema Trace.Wri var vertexTypes = graph.FindVertexTypes(); //vertexTypes.Select(x => x.TypeName).PrintDump(); var edgeTypes = graph.FindEdgeTypes(); VertexType concept = vertexTypes.FirstOrDefault(x => x.TypeName == "Concept") ?? graph.NewVertexType("Concept"); PropertyType conceptName = concept.FindProperty("ConceptName"); Assert.IsNotNull(conceptName, "ConceptName"); PropertyType conceptSize = concept.FindProperty("ConceptSize"); Assert.IsNotNull(conceptSize, "ConceptSize"); PropertyType conceptFrequency = concept.FindProperty("ConceptFrequency"); //PropertyType similarity = concept.NewProperty("Similarity", VelocityGraph.DataType.Double, // VelocityGraph.PropertyKind.NotIndexed); PropertyType vagueness = concept.FindProperty("Vagueness");//, DataType.Double,PropertyKind.NotIndexed); VertexType instance = vertexTypes.FirstOrDefault(x => x.TypeName == "Instance"); PropertyType instanceSize = instance.FindProperty("InstanceSize"); PropertyType instanceName = instance.FindProperty("InstanceName"); PropertyType instanceFrequency = instance.FindProperty("InstanceFrequency"); //VertexType attributes = graph.NewVertexType("Attribute"); ////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); //EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); //PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, // PropertyKind.NotIndexed); //VertexType synonym = graph.NewVertexType("Synonym", namedScore); //PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); //EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = edgeTypes.FirstOrDefault(x => x.TypeName == "IsA"); PropertyType frequency = isA.FindProperty("frequency"); PropertyType popularity = isA.FindProperty("popularity"); PropertyType ZipfSlope = isA.FindProperty("zipf_slope"); PropertyType ZipfPearson = isA.FindProperty("zipf_pearson"); EdgeType cooccurence = graph.FindEdgeType("Coocurrence"); //LRVertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //LRVertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //LREdge edge = graph.NewEdge(cooccurence, vx1, vx2); Assert.IsNotNull(conceptName); Vertex f = graph.FindVertex(conceptName, "factor"); var inst = graph.FindVertex(instanceName, "age"); Assert.IsNotNull(f); Assert.IsNotNull(inst); } //if (instanceVertex == null) }
// [Test] public void TestVelecoityBuildLocal() { using (var session = new SessionNoServer(@"d:\graphtest")) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4; DataCache.MaximumMemoryUse = 2000000000; //var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0, // 3, session); //DatabaseLocation bl = session.NewLocation(dbl); //session.Commit(false); // Assert.That(bl!=null); var graph = new Graph(session); session.Persist(graph); //define schema // session.BeginUpdate(); VertexType namedScore = graph.NewVertexType("NamedScore"); PropertyType name = namedScore.NewProperty("Name", DataType.String, PropertyKind.Indexed); PropertyType int_score = namedScore.NewProperty("IntScore", DataType.Integer, PropertyKind.NotIndexed); PropertyType double_score = namedScore.NewProperty("DoubleScore", DataType.Double, PropertyKind.NotIndexed); VertexType concept = graph.NewVertexType("Concept"); PropertyType conceptName = concept.NewProperty("Name", DataType.String, PropertyKind.Unique); PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed); PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed); VertexType instance = graph.NewVertexType("Instance", concept); PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType instanceName = instance.NewProperty("Name", DataType.String, PropertyKind.Unique); VertexType attributes = graph.NewVertexType("Attribute", namedScore); //EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,); EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance); PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, PropertyKind.NotIndexed); VertexType synonym = graph.NewVertexType("Synonym", namedScore); PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed); EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance); EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance); PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed); PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed); PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed); PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed); EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept); //Vertex vx1 = graph.NewVertex(instance); //vx1.SetProperty("Name", "bla"); //Vertex vx2 = graph.NewVertex(instance); //vx2.SetProperty("bla", "name"); //Edge edge = graph.NewEdge(cooccurence, vx1, vx2); using (var llz = new StreamReader(@"d:\isa_core.txt")) { string lastConcept = string.Empty; while (!llz.EndOfStream) { string ln = llz.ReadLine(); if (string.IsNullOrEmpty(ln)) continue; string[] items = ln.Split(new[] { '\t' }); if (items.Length < 4) continue; int id = -1; if (!int.TryParse(items[2], out id)) continue; var conceptVertex = graph.FindVertex(conceptName, items[0], false); if (conceptVertex == null) { conceptVertex = graph.NewVertex(concept); conceptVertex.SetProperty(conceptName, items[0]); conceptVertex.SetProperty(conceptFrequency, int.Parse(items[4])); conceptVertex.SetProperty(conceptSize, int.Parse(items[5])); double d = double.NaN; double.TryParse(items[6], out d); conceptVertex.SetProperty(vagueness, d); d = double.NaN; double.TryParse(items[7], out d); conceptVertex.SetProperty(ZipfSlope, d); d = double.NaN; double.TryParse(items[8], out d); conceptVertex.SetProperty(ZipfPearson, d); } var instanceVertex = graph.FindVertex(instanceName, items[1], false); if (instanceVertex == null) { instanceVertex = graph.NewVertex(instance); instanceVertex.SetProperty(instanceFrequency, int.Parse(items[9])); instanceVertex.SetProperty(instanceSize, int.Parse(items[10])); } var isaedge = graph.NewEdge(isA, conceptVertex, instanceVertex); isaedge.SetProperty(frequency, int.Parse(items[2])); isaedge.SetProperty(popularity, int.Parse(items[3])); } session.Commit(); } } }
public void Create1Vertices(bool vertexIdSetPerVertexType) { DataCache.MaximumMemoryUse = 10000000000; // 10 GB bool dirExist = Directory.Exists(systemDir); try { if (Directory.Exists(systemDir)) Directory.Delete(systemDir, true); // remove systemDir from prior runs and all its databases. Directory.CreateDirectory(systemDir); File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); } catch { File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); } using (SessionNoServer session = new SessionNoServer(systemDir, 5000, false, true)) { session.BeginUpdate(); Graph g = new Graph(session, vertexIdSetPerVertexType); session.Persist(g); VertexType userType = g.NewVertexType("User"); VertexType otherType = g.NewVertexType("Other"); PropertyType userNamePropertyType = g.NewVertexProperty(userType, "NAME", DataType.String, PropertyKind.Indexed); VertexType powerUserType = g.NewVertexType("PowerUser", userType); EdgeType userFriendEdgeType = g.NewEdgeType("Friend", true, userType, userType); EdgeType userBestFriendEdgeType = g.NewEdgeType("Best Friend", true, userType, userType, userFriendEdgeType); EdgeType otherEdgeType = g.NewEdgeType("Other", true, userType, userType); PropertyType bestFriendPropertyType = g.NewEdgeProperty(userFriendEdgeType, "START", DataType.DateTime, PropertyKind.Indexed); Vertex kinga = userType.NewVertex(); Vertex robin = userType.NewVertex(); Vertex mats = powerUserType.NewVertex(); Vertex chiran = powerUserType.NewVertex(); Vertex other = otherType.NewVertex(); Edge bestFriend = kinga.AddEdge(userBestFriendEdgeType, robin); Edge otherEdge = kinga.AddEdge(otherEdgeType, robin); DateTime now = DateTime.UtcNow; mats.SetProperty("Address", 1); bestFriend.SetProperty(bestFriendPropertyType, now); kinga.SetProperty(userNamePropertyType, "Kinga"); if (g.VertexIdSetPerType == false) mats.SetProperty(userNamePropertyType, "Mats"); else { try { mats.SetProperty(userNamePropertyType, "Mats"); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } } try { other.SetProperty(userNamePropertyType, "Mats"); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } try { otherEdge.SetProperty(bestFriendPropertyType, now); Assert.Fail("Invalid property for VertexType not handled"); } catch (Exception) { } Vertex findMats = userNamePropertyType.GetPropertyVertex("Mats", true); var list = userNamePropertyType.GetPropertyVertices("Mats", true).ToList(); //Edge findWhen = bestFriendPropertyType.GetPropertyEdge(now); //var list2 = bestFriendPropertyType.GetPropertyEdges(now); Console.WriteLine(findMats); // session.Commit(); // session.BeginRead(); PropertyType adressProperty = g.FindVertexProperty(powerUserType, "Address"); Vertex find1 = adressProperty.GetPropertyVertex(1, true); session.Abort(); /*session.BeginUpdate(); g.Unpersist(session); session.Commit(); dirExist = Directory.Exists(systemDir); try { if (Directory.Exists(systemDir)) Directory.Delete(systemDir, true); // remove systemDir from prior runs and all its databases. Directory.CreateDirectory(systemDir); File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); } catch { File.Copy(licenseDbFile, Path.Combine(systemDir, "4.odb")); }*/ } using (SessionNoServer session = new SessionNoServer(systemDir, 5000, false, true)) { session.BeginUpdate(); session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.None; Graph g = new Graph(session, vertexIdSetPerVertexType); session.Persist(g); Graph g2 = new Graph(session); session.Persist(g2); Graph g3 = new Graph(session); session.Persist(g3); UInt32 dbNum = session.DatabaseNumberOf(typeof(Graph)); Graph g4 = (Graph)session.Open(dbNum, 2, 1, true); // g4 == g Graph g5 = (Graph)session.Open(dbNum, 2, 2, true); // g5 == g2 Graph g6 = (Graph)session.Open(dbNum, 2, 3, true); // g6 == g3 for (int i = 4; i < 8; i++) { Graph gt = new Graph(session); session.Persist(gt); } Graph g7 = new Graph(session); Placement place = new Placement(dbNum, 15); session.Persist(place, g7); // SCHEMA VertexType userType = g.NewVertexType("User"); VertexType locationType = g.NewVertexType("Location"); VertexType aVertexType = g.NewVertexType("A"); VertexType bVertexType = g.NewVertexType("B"); VertexType cVertexType = g.NewVertexType("C"); EdgeType uEdge = g.NewEdgeType("unrestricted", true); Vertex aVertex = g.NewVertex(aVertexType); Vertex bVertex = g.NewVertex(bVertexType); Vertex cVertex = g.NewVertex(cVertexType); Edge abEdge = (Edge)aVertex.AddEdge("unrestricted", bVertex); Edge bcEdge = (Edge)aVertex.AddEdge("unrestricted", cVertex); Dictionary<Vertex, HashSet<Edge>> traverse = aVertex.Traverse(uEdge, Direction.Out); abEdge.Remove(); Dictionary<Vertex, HashSet<Edge>> traverse2 = aVertex.Traverse(uEdge, Direction.Out); EdgeType friendEdgeType = g.NewEdgeType("Friend", true, userType, userType); EdgeType userLocationEdgeType = g.NewEdgeType("UserLocation", true, userType, locationType); // DATA Random rand = new Random(5); for (int i = 0; i < numberOfUserVertices / 100; i++) { int vId = rand.Next(numberOfUserVertices); try { if (g.VertexIdSetPerType) userType.GetVertex(vId); else g.GetVertex(vId); try { userType.NewVertex(vId); Assert.Fail(); } catch (VertexAllreadyExistException) { } } catch (VertexDoesNotExistException) { userType.NewVertex(vId); userType.GetVertex(vId); } } for (int i = 0; i < numberOfUserVertices / 10000; i++) { int vId = rand.Next(numberOfUserVertices); try { Vertex v = userType.GetVertex(vId); v.SetProperty("test", 1); } catch (VertexDoesNotExistException) { } } for (int i = 0; i < numberOfUserVertices / 10000; i++) { int vId = rand.Next(numberOfUserVertices); try { Vertex v = userType.GetVertex(vId); userType.RemoveVertex(v); } catch (VertexDoesNotExistException) { } } foreach (Vertex v in userType.GetVertices().ToArray()) userType.RemoveVertex(v); Assert.AreEqual(0, userType.GetVertices().Count()); for (int i = 100000; i < numberOfUserVertices; i++) userType.NewVertex(); for (int i = 1; i < 100000; i++) userType.NewVertex(); for (int i = 1; i < numberOfLocationVertices; i++) locationType.NewVertex(); session.Commit(); session.BeginRead(); foreach (var x in session.AllObjects<BTreeSet<Range<VertexId>>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeSet<EdgeType>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeSet<EdgeIdVertexId>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<EdgeId, VelocityDbList<ElementId>>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<string, PropertyType>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<string, EdgeType>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<string, VertexType>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<VertexId, BTreeSet<EdgeIdVertexId>>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<VertexType, BTreeMap<VertexId, BTreeSet<EdgeIdVertexId>>>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); foreach (var x in session.AllObjects<BTreeMap<EdgeType, BTreeMap<VertexType, BTreeMap<VertexId, BTreeSet<EdgeIdVertexId>>>>>(false, true)) Assert.True(x.ToDoBatchAddCount == 0); session.Commit(); Validate(); } }