static void Main(string[] args)
{
bool import = args.Length > 0 && args[0].ToLower() == "-import";
bool dirExist = Directory.Exists(s_systemDir);
SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false;
if (import || !dirExist)
{
if (dirExist)
{
Directory.Delete(s_systemDir, true); // remove systemDir from prior runs and all its databases.
}
Directory.CreateDirectory(s_systemDir);
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
DataCache.MaximumMemoryUse = 12000000000; // 12 GB, set this to what fits your case
SessionBase.BTreeAddFastTransientBatchSize = 10; // reduces memory usage
Vertex[] ratingVertices = new Vertex[10];
session.BeginUpdate();
session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4;
Graph g = new Graph(session);
// SCHEMA
VertexType userType = g.NewVertexType("User");
PropertyType genderType = userType.NewProperty("Gender", DataType.Integer, PropertyKind.Indexed);
VertexType ratingType = g.NewVertexType("Rating");
PropertyType ratingValuePropertyType = ratingType.NewProperty("RatingValue", DataType.Integer, PropertyKind.Indexed);
EdgeType ratingEdgeType = g.NewEdgeType("UserToRating", true, userType, ratingType);
EdgeType ratingOfType = g.NewEdgeType("RatingOf", false, userType, userType);
PropertyType ratingEdgePropertyType = ratingOfType.NewProperty("Rating", DataType.Integer, PropertyKind.Indexed);
// DATA
using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "gender.dat")))
{
using (StreamReader file = new System.IO.StreamReader(stream))
{
string line;
int lineNumber = 0;
while ((line = file.ReadLine()) != null)
{
lineNumber++;
string[] fields = line.Split(',');
Vertex aUser = userType.NewVertex();
aUser.SetProperty(genderType, (int)fields[1][0] == 'M' ? Gender.Male : fields[1][0] == 'F' ? Gender.Female : Gender.Unknown);
}
Console.WriteLine("Done importing " + lineNumber + " users");
}
}
using (FileStream stream = File.OpenRead(System.IO.Path.Combine(s_inputDataDir, "ratings.dat")))
{
using (StreamReader file = new System.IO.StreamReader(stream))
{
string line;
int lineNumber = 0;
Vertex rater = null;
int raterId;
int priorRaterId = -1;
while ((line = file.ReadLine()) != null)
{
lineNumber++;
if (lineNumber % 1000000 == 0)
{
Console.WriteLine("Parsing rating # " + lineNumber);
}
string[] fields = line.Split(',');
raterId = int.Parse(fields[0]);
if (raterId != priorRaterId)
{
rater = userType.GetVertex(raterId);
}
priorRaterId = raterId;
int ratedId = int.Parse(fields[1]);
int rating = int.Parse(fields[2]);
Vertex ratingVertex = ratingVertices[rating - 1];
if (ratingVertex == null)
{
ratingVertex = ratingType.NewVertex();
ratingVertex.SetProperty(ratingValuePropertyType, rating);
ratingVertices[rating - 1] = ratingVertex;
}
Vertex rated = userType.GetVertex(ratedId);
Edge aRatingOf = ratingOfType.NewEdge(rater, rated);
aRatingOf.SetProperty(ratingEdgePropertyType, rating);
Edge userRating = ratingEdgeType.NewEdge(rated, ratingVertex);
}
Console.WriteLine("Done importing " + lineNumber + " ratings");
}
}
session.Commit();
}
}
// Query
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
session.BeginRead();
Graph g = Graph.Open(session);
VertexType userType = g.FindVertexType("User");
PropertyType genderType = userType.FindProperty("Gender");
VertexType ratingType = g.FindVertexType("Rating");
PropertyType ratingValuePropertyType = ratingType.FindProperty("RatingValue");
EdgeType ratingEdgeType = g.FindEdgeType("UserToRating");
EdgeType ratingOfType = g.FindEdgeType("RatingOf");
PropertyType ratingEdgePropertyType = ratingOfType.FindProperty("Rating");
// Complex queries
int ct = 0;
// Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated
Vertex someUser = userType.GetVertex(1);
var similarRatings = (from Edge e in someUser.GetEdges(ratingOfType, Direction.Out)
from Edge edge in e.Head.GetEdges(ratingOfType, Direction.Out)
where someUser != edge.Head
where ((int)e.GetProperty(ratingEdgePropertyType) == (int)edge.GetProperty(ratingEdgePropertyType))
select edge.Tail).Distinct();
var someUserRated = from Edge e in someUser.GetEdges(ratingOfType, Direction.Out)
select e.Head;
var recommendedProfles = from v in similarRatings
from Edge e in v.GetEdges(ratingOfType, Direction.Out)
where someUserRated.Contains(e.Head) == false
select e.Head;
Console.WriteLine("Some user's rated profiles");
ct = 0;
foreach (Vertex v in someUserRated)
{
if (ct++ % 50 == 0) // don't print them all !
{
Console.WriteLine("User id: " + v.VertexId);
}
}
Console.WriteLine("Number of some user's rated profiles: " + ct);
Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles");
ct = 0;
foreach (Vertex v in similarRatings)
{
if (ct++ % 50 == 0) // don't print them all !
{
Console.WriteLine("User id: " + v.VertexId);
}
}
Console.WriteLine("Number of matching profiles: " + ct);
Console.WriteLine("Given a user id, and based on the rated profiles, find other users who have rated similarly on the same profiles, and find other profiles to recommend based on what those other users have rated");
ct = 0;
foreach (Vertex v in recommendedProfles)
{
if (ct++ % 5000 == 0) // don't print them all !
{
Console.WriteLine("User id: " + v.VertexId);
}
}
Console.WriteLine("Number of matching profiles: " + ct);
// Get all female users with less than 50 ratings
Console.WriteLine();
var females = from u in userType.GetVertices()
where ((Gender)u.GetProperty(genderType)) == Gender.Female
select u;
var femalesLess50Ratings = from f in females
where f.GetNumberOfEdges(ratingEdgeType, Direction.Out) < 50
select f;
Console.WriteLine("Female users with less than 50 ratings");
ct = 0;
foreach (Vertex f in femalesLess50Ratings)
{
long count = f.GetNumberOfEdges(ratingEdgeType, Direction.Out);
if (ct++ % 5000 == 0) // don't print them all !
{
Console.WriteLine("User id: " + f.VertexId + "\tnumber of ratings: " + count);
}
}
Console.WriteLine("Number of females with fewer than 50 ratings: " + ct);
Console.WriteLine();
// Get all male users with at least one 10 rating
Console.WriteLine();
var rated10vertex = (from v in ratingType.GetVertices()
where ((int)v.GetProperty(ratingValuePropertyType)) == 10
select v).First();
var rated10 = (from e in rated10vertex.GetEdges(ratingEdgeType, Direction.In)
select e.GetVertex(Direction.Out)).Distinct();
var rated10male = from Vertex v in rated10
where ((Gender)v.GetProperty(genderType)) == Gender.Male
select v;
Console.WriteLine("Males with at least one 10 rating");
ct = 0;
foreach (Vertex v in rated10male)
{
if (ct++ % 5000 == 0) // don't print them all !
{
Console.WriteLine("User id: " + v.VertexId);
}
}
Console.WriteLine("Number of males with at least one 10 rating: " + ct);
Console.WriteLine();
// Get the first 10 male users who have rated at least 3 of the same profiles as the given user.
Console.WriteLine("10 male users who have rated at least 3 of the same profiles as the given user");
var males = from u in userType.GetVertices()
where ((Gender)u.GetProperty(genderType)) == Gender.Male
select u;
var someUserHasRated = from Edge o in someUser.GetEdges(ratingOfType, Direction.Out)
select o.Head;
var first10withSame3ratedAs = from m in males
where (from Edge r in m.GetEdges(ratingOfType, Direction.Out) select r.Head).Intersect(someUserHasRated).ToArray().Length >= 3
select m;
ct = 0;
foreach (Vertex v in first10withSame3ratedAs)
{
if (++ct > 10)
{
break;
}
Console.WriteLine("User id: " + v.VertexId);
}
Console.WriteLine();
// Statistical queries
// Get the 20 profiles with the most ratings
var top20mostRatings = (from v in userType.GetVertices()
orderby v.GetNumberOfEdges(ratingEdgeType, Direction.Out) descending
select v).Take(20);
Console.WriteLine("20 profiles with the most ratings");
ct = 0;
foreach (Vertex v in top20mostRatings)
{
int count = (int)v.GetNumberOfEdges(ratingEdgeType, Direction.Out);
Console.WriteLine("User id: " + v.VertexId + "\tnumber of ratings: " + count);
}
Console.WriteLine();
var ratingsVertexEnum = from v in ratingType.GetVertices() orderby v.GetProperty(ratingValuePropertyType) descending select v;
Vertex rating10Vertex = ratingsVertexEnum.First();
// Get the 20 best rated profiles regardless of gender
var top = from u in userType.GetVertices()
let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out)
orderby edgeCt descending
select new { u, edgeCt };
Console.WriteLine("20 best rated profiles regardless of gender");
ct = 0;
foreach (var v in top)
{
if (++ct > 20)
{
break;
}
Console.WriteLine("User id: " + v.u.VertexId + "\t10 ratings: " + v.edgeCt);
}
Console.WriteLine();
// Get the 20 best rated males
Console.WriteLine("20 best rated male profiles");
var top20males = from u in userType.GetVertices()
where ((Gender)u.GetProperty(genderType)) == Gender.Male
let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out)
orderby edgeCt descending
select new { u, edgeCt };
ct = 0;
foreach (var v in top20males)
{
if (++ct > 20)
{
break;
}
Console.WriteLine("Male User id: " + v.u.VertexId + " \t10 ratings: " + v.edgeCt);
}
Console.WriteLine();
// Get the 20 best rated females
Console.WriteLine("20 best rated female profiles");
var top20females = from u in userType.GetVertices()
where ((Gender)u.GetProperty(genderType)) == Gender.Female
let edgeCt = u.GetNumberOfEdges(ratingEdgeType, rating10Vertex, Direction.Out)
orderby edgeCt descending
select new { u, edgeCt };
ct = 0;
foreach (var v in top20females)
{
if (++ct > 20)
{
break;
}
Console.WriteLine("Female User id: " + v.u.VertexId + "\t10 ratings: " + v.edgeCt);
}
session.Commit();
}
}
static readonly string s_systemDir = "SupplierTracking"; // appended to SessionBase.BaseDatabasePath
static int Main(string[] args)
{
DataCache.MaximumMemoryUse = 10000000000; // 10 GB, set this to what fits your case
SupplierTracking supplierTracking = new SupplierTracking();
using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, true))
{
if (Directory.Exists(session.SystemDirectory))
{
Directory.Delete(session.SystemDirectory, true); // remove systemDir from prior runs and all its databases.
}
Directory.CreateDirectory(session.SystemDirectory);
session.BeginUpdate();
Graph g = new Graph(session, false);
// SCHEMA
VertexType warehouseVertexType = g.NewVertexType("Warehouse");
VertexType supplierVertexType = g.NewVertexType("Supplier");
EdgeType supplierWarehouseEdgeType = g.NewEdgeType("Warehouse", true, supplierVertexType, warehouseVertexType);
EdgeType moveToS1EdgeType = g.NewEdgeType("MoveS1To", true, warehouseVertexType, warehouseVertexType);
EdgeType moveToS2EdgeType = g.NewEdgeType("MoveS2To", true, warehouseVertexType, warehouseVertexType);
EdgeType moveToS3EdgeType = g.NewEdgeType("MoveS3To", true, warehouseVertexType, warehouseVertexType);
PropertyType supplierNameProperty = supplierVertexType.NewProperty("name", DataType.String, PropertyKind.NotIndexed);
PropertyType wareHouseNameProperty = warehouseVertexType.NewProperty("name", DataType.String, PropertyKind.NotIndexed);
PropertyType howManyS1Property = moveToS1EdgeType.NewProperty("howMany", DataType.Integer, PropertyKind.NotIndexed);
PropertyType howManyS2Property = moveToS2EdgeType.NewProperty("howMany", DataType.Integer, PropertyKind.NotIndexed);
PropertyType howManyS3Property = moveToS3EdgeType.NewProperty("howMany", DataType.Integer, PropertyKind.NotIndexed);
Vertex supplier1 = supplierVertexType.NewVertex();
supplier1.SetProperty(supplierNameProperty, "S1");
Vertex supplier2 = supplierVertexType.NewVertex();
supplier2.SetProperty(supplierNameProperty, "S2");
Vertex supplier3 = supplierVertexType.NewVertex();
supplier3.SetProperty(supplierNameProperty, "S3");
Vertex wareHouse1Supplier1 = warehouseVertexType.NewVertex();
supplier1.AddEdge(supplierWarehouseEdgeType, wareHouse1Supplier1);
wareHouse1Supplier1.SetProperty(wareHouseNameProperty, "A01");
Vertex wareHouse2Supplier1 = warehouseVertexType.NewVertex();
supplier1.AddEdge(supplierWarehouseEdgeType, wareHouse2Supplier1);
wareHouse2Supplier1.SetProperty(wareHouseNameProperty, "A02");
Vertex wareHouse3Supplier1 = warehouseVertexType.NewVertex();
supplier1.AddEdge(supplierWarehouseEdgeType, wareHouse3Supplier1);
wareHouse3Supplier1.SetProperty(wareHouseNameProperty, "A05");
Vertex wareHouse4Supplier1 = warehouseVertexType.NewVertex();
supplier1.AddEdge(supplierWarehouseEdgeType, wareHouse4Supplier1);
wareHouse4Supplier1.SetProperty(wareHouseNameProperty, "A06");
Vertex wareHouse1Supplier2 = warehouseVertexType.NewVertex();
supplier2.AddEdge(supplierWarehouseEdgeType, wareHouse1Supplier2);
wareHouse1Supplier2.SetProperty(wareHouseNameProperty, "A03");
Vertex wareHouse2Supplier2 = warehouseVertexType.NewVertex();
supplier2.AddEdge(supplierWarehouseEdgeType, wareHouse2Supplier2);
wareHouse2Supplier2.SetProperty(wareHouseNameProperty, "A07");
Vertex wareHouse1Supplier3 = warehouseVertexType.NewVertex();
supplier3.AddEdge(supplierWarehouseEdgeType, wareHouse1Supplier3);
wareHouse1Supplier3.SetProperty(wareHouseNameProperty, "A04");
Vertex wareHouse2Supplier3 = warehouseVertexType.NewVertex();
supplier3.AddEdge(supplierWarehouseEdgeType, wareHouse2Supplier3);
wareHouse2Supplier3.SetProperty(wareHouseNameProperty, "A08");
Vertex wareHouseB1 = warehouseVertexType.NewVertex();
wareHouseB1.SetProperty(wareHouseNameProperty, "B01");
Vertex wareHouseB2 = warehouseVertexType.NewVertex();
wareHouseB2.SetProperty(wareHouseNameProperty, "B02");
Vertex wareHouseB3 = warehouseVertexType.NewVertex();
wareHouseB3.SetProperty(wareHouseNameProperty, "B03");
Vertex wareHouseB4 = warehouseVertexType.NewVertex();
wareHouseB4.SetProperty(wareHouseNameProperty, "B04");
Vertex wareHouseC1 = warehouseVertexType.NewVertex();
wareHouseC1.SetProperty(wareHouseNameProperty, "C01");
Vertex wareHouseC2 = warehouseVertexType.NewVertex();
wareHouseC2.SetProperty(wareHouseNameProperty, "C02");
Vertex wareHouseD1 = warehouseVertexType.NewVertex();
wareHouseD1.SetProperty(wareHouseNameProperty, "D01");
Edge moveA1toB1 = wareHouse1Supplier1.AddEdge(moveToS1EdgeType, wareHouseB1);
moveA1toB1.SetProperty(howManyS1Property, 750);
Edge moveA2toB1 = wareHouse2Supplier1.AddEdge(moveToS1EdgeType, wareHouseB1);
moveA2toB1.SetProperty(howManyS1Property, 500);
Edge moveA3toB2 = wareHouse1Supplier2.AddEdge(moveToS2EdgeType, wareHouseB2);
moveA3toB2.SetProperty(howManyS2Property, 750);
Edge moveA4toB2 = wareHouse1Supplier3.AddEdge(moveToS3EdgeType, wareHouseB2);
moveA4toB2.SetProperty(howManyS3Property, 500);
Edge moveA5toB3 = wareHouse3Supplier1.AddEdge(moveToS1EdgeType, wareHouseB3);
moveA5toB3.SetProperty(howManyS1Property, 100);
Edge moveA6toB3 = wareHouse4Supplier1.AddEdge(moveToS1EdgeType, wareHouseB3);
moveA6toB3.SetProperty(howManyS1Property, 200);
Edge moveA7toB4 = wareHouse2Supplier2.AddEdge(moveToS2EdgeType, wareHouseB4);
moveA7toB4.SetProperty(howManyS2Property, 50);
Edge moveA8toB4 = wareHouse2Supplier3.AddEdge(moveToS3EdgeType, wareHouseB4);
moveA8toB4.SetProperty(howManyS3Property, 450);
Edge moveB1toC1 = wareHouseB1.AddEdge(moveToS1EdgeType, wareHouseC1);
moveB1toC1.SetProperty(howManyS1Property, 400);
Edge moveS2B2toC1 = wareHouseB2.AddEdge(moveToS2EdgeType, wareHouseC1);
moveS2B2toC1.SetProperty(howManyS2Property, 360);
Edge moveS3B2toC1 = wareHouseB2.AddEdge(moveToS3EdgeType, wareHouseC1);
moveS3B2toC1.SetProperty(howManyS3Property, 240);
Edge moveS1B3toC2 = wareHouseB3.AddEdge(moveToS1EdgeType, wareHouseC2);
moveS1B3toC2.SetProperty(howManyS1Property, 100);
Edge moveS2B4toC2 = wareHouseB4.AddEdge(moveToS2EdgeType, wareHouseC2);
moveS2B4toC2.SetProperty(howManyS2Property, 20);
Edge moveS3B4toC2 = wareHouseB4.AddEdge(moveToS3EdgeType, wareHouseC2);
moveS3B4toC2.SetProperty(howManyS3Property, 180);
Edge moveS1C1toD1 = wareHouseC1.AddEdge(moveToS1EdgeType, wareHouseD1);
moveS1C1toD1.SetProperty(howManyS1Property, 200);
Edge moveS2C1toD1 = wareHouseC1.AddEdge(moveToS2EdgeType, wareHouseD1);
moveS2C1toD1.SetProperty(howManyS2Property, 180);
Edge moveS3C1toD1 = wareHouseC1.AddEdge(moveToS3EdgeType, wareHouseD1);
moveS3C1toD1.SetProperty(howManyS3Property, 120);
Edge moveS1C2toD1 = wareHouseC2.AddEdge(moveToS1EdgeType, wareHouseD1);
moveS1C2toD1.SetProperty(howManyS1Property, 67);
Edge moveS2C2toD1 = wareHouseC2.AddEdge(moveToS2EdgeType, wareHouseD1);
moveS2C2toD1.SetProperty(howManyS2Property, 13);
Edge moveS3C2toD1 = wareHouseC2.AddEdge(moveToS3EdgeType, wareHouseD1);
moveS3C2toD1.SetProperty(howManyS3Property, 120);
Console.WriteLine("Supplier 1 to Warehouse D total: " + supplierTracking.CalculateTotalTo(supplier1, supplierWarehouseEdgeType, moveToS1EdgeType, howManyS1Property, wareHouseD1));
Console.WriteLine("Supplier 2 to Warehouse D total: " + supplierTracking.CalculateTotalTo(supplier2, supplierWarehouseEdgeType, moveToS2EdgeType, howManyS2Property, wareHouseD1));
Console.WriteLine("Supplier 3 to Warehouse D total: " + supplierTracking.CalculateTotalTo(supplier3, supplierWarehouseEdgeType, moveToS3EdgeType, howManyS3Property, wareHouseD1));
VelocityGraph.GraphJsonSettings gs = new VelocityGraph.GraphJsonSettings();
gs.ClusterFuncProp = v =>
{
string s = (string)v.GetProperty("name");
if (s == null)
{
return(0);
}
return(Convert.ToInt32(s.First()));
};
gs.VertexTypeFuncProp = v =>
{
Vertex vertex = v as Vertex;
string s = (string)vertex.VertexType.TypeName;
return(s);
};
//g.ExportToGraphJson("c:/SupplierTrackingExportToGraphJson.json", gs);
Graph g2 = new Graph(session);
//g2.ImportGraphJson("c:/SupplierTrackingExportToGraphJson.json");
session.Commit();
return(0);
}
}
// [Test]
public void TestVelecoityBuildLocal()
{
using (var session = new SessionNoServer(@"d:\graphtest"))
{
session.BeginUpdate();
session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4;
DataCache.MaximumMemoryUse = 2000000000;
//var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0,
// 3, session);
//DatabaseLocation bl = session.NewLocation(dbl);
//session.Commit(false);
// Assert.That(bl!=null);
var graph = new Graph(session);
session.Persist(graph);
//define schema
// session.BeginUpdate();
VertexType namedScore = graph.NewVertexType("NamedScore");
PropertyType name = namedScore.NewProperty("Name", DataType.String, PropertyKind.Indexed);
PropertyType int_score = namedScore.NewProperty("IntScore", DataType.Integer,
PropertyKind.NotIndexed);
PropertyType double_score = namedScore.NewProperty("DoubleScore", DataType.Double,
PropertyKind.NotIndexed);
VertexType concept = graph.NewVertexType("Concept");
PropertyType conceptName = concept.NewProperty("Name", DataType.String, PropertyKind.Unique);
PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed);
PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed);
PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed);
PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed);
VertexType instance = graph.NewVertexType("Instance", concept);
PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer, PropertyKind.NotIndexed);
PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency", DataType.Integer, PropertyKind.NotIndexed);
PropertyType instanceName = instance.NewProperty("Name", DataType.String, PropertyKind.Unique);
VertexType attributes = graph.NewVertexType("Attribute", namedScore);
//EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,);
EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance);
PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer, PropertyKind.NotIndexed);
VertexType synonym = graph.NewVertexType("Synonym", namedScore);
PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed);
EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance);
EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance);
PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed);
PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed);
PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed);
PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed);
EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept);
//Vertex vx1 = graph.NewVertex(instance);
//vx1.SetProperty("Name", "bla");
//Vertex vx2 = graph.NewVertex(instance);
//vx2.SetProperty("bla", "name");
//Edge edge = graph.NewEdge(cooccurence, vx1, vx2);
using (var llz = new StreamReader(@"d:\isa_core.txt"))
{
string lastConcept = string.Empty;
while (!llz.EndOfStream)
{
string ln = llz.ReadLine();
if (string.IsNullOrEmpty(ln))
{
continue;
}
string[] items = ln.Split(new[] { '\t' });
if (items.Length < 4)
{
continue;
}
int id = -1;
if (!int.TryParse(items[2], out id))
{
continue;
}
var conceptVertex = graph.FindVertex(conceptName, items[0], false);
if (conceptVertex == null)
{
conceptVertex = graph.NewVertex(concept);
conceptVertex.SetProperty(conceptName, items[0]);
conceptVertex.SetProperty(conceptFrequency, int.Parse(items[4]));
conceptVertex.SetProperty(conceptSize, int.Parse(items[5]));
double d = double.NaN;
double.TryParse(items[6], out d);
conceptVertex.SetProperty(vagueness, d);
d = double.NaN;
double.TryParse(items[7], out d);
conceptVertex.SetProperty(ZipfSlope, d);
d = double.NaN;
double.TryParse(items[8], out d);
conceptVertex.SetProperty(ZipfPearson, d);
}
var instanceVertex = graph.FindVertex(instanceName, items[1], false);
if (instanceVertex == null)
{
instanceVertex = graph.NewVertex(instance);
instanceVertex.SetProperty(instanceFrequency, int.Parse(items[9]));
instanceVertex.SetProperty(instanceSize, int.Parse(items[10]));
}
var isaedge = graph.NewEdge(isA, conceptVertex, instanceVertex);
isaedge.SetProperty(frequency, int.Parse(items[2]));
isaedge.SetProperty(popularity, int.Parse(items[3]));
}
session.Commit();
}
}
}
public void AddVertices()
{
using (var session = new SessionNoServer(@"d:\graphtest2"))
{
session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4;
DataCache.MaximumMemoryUse = 4000000000;
var sw = new Stopwatch();
sw.Start();
//int i = 0;
//var dbl = new DatabaseLocation(Dns.GetHostEntry("wordanalysis.cloudapp.net").HostName, @"Z:\DBStore\", 0,
// 3, session);
//DatabaseLocation bl = session.NewLocation(dbl);
//session.Commit(false);
// Assert.That(bl!=null);
// var db = session.OpenDatabase(15, true);
var graph = new Graph(session);
session.BeginUpdate();
session.Persist(graph);
//define schema Trace.Wri
VertexType concept = graph.NewVertexType("Concept");
PropertyType conceptName = concept.NewProperty("ConceptName", DataType.String, PropertyKind.Unique);
PropertyType conceptSize = concept.NewProperty("ConceptSize", DataType.Integer, PropertyKind.NotIndexed);
PropertyType conceptFrequency = concept.NewProperty("ConceptFrequency", DataType.Integer, PropertyKind.NotIndexed);
PropertyType similarity = concept.NewProperty("Similarity", DataType.Double, PropertyKind.NotIndexed);
PropertyType vagueness = concept.NewProperty("Vagueness", DataType.Double, PropertyKind.NotIndexed);
VertexType instance = graph.NewVertexType("Instance", concept);
PropertyType instanceSize = instance.NewProperty("InstanceSize", DataType.Integer,
PropertyKind.NotIndexed);
PropertyType instanceName = instance.NewProperty("InstanceName", DataType.String,
PropertyKind.Unique);
PropertyType instanceFrequency = instance.NewProperty("InstanceFrequency",
DataType.Integer,
PropertyKind.NotIndexed);
//VertexType attributes = graph.NewVertexType("Attribute");
////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,);
//EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance);
//PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer,
// PropertyKind.NotIndexed);
//VertexType synonym = graph.NewVertexType("Synonym", namedScore);
//PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed);
//EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance);
EdgeType isA = graph.NewEdgeType("IsA", true, concept, instance);
PropertyType frequency = isA.NewProperty("frequency", DataType.Integer, PropertyKind.NotIndexed);
PropertyType popularity = isA.NewProperty("popularity", DataType.Integer, PropertyKind.NotIndexed);
PropertyType ZipfSlope = isA.NewProperty("zipf_slope", DataType.Double, PropertyKind.NotIndexed);
PropertyType ZipfPearson = isA.NewProperty("zipf_pearson", DataType.Double, PropertyKind.NotIndexed);
EdgeType cooccurence = graph.NewEdgeType("Coocurrence", true, concept, concept);
//LRVertex vx1 = graph.NewVertex(instance);
//vx1.SetProperty("Name", "bla");
//LRVertex vx2 = graph.NewVertex(instance);
//vx2.SetProperty("bla", "name");
//LREdge edge = graph.NewEdge(cooccurence, vx1, vx2);
Vertex v2 = graph.NewVertex(concept);
v2.SetProperty(conceptName, "factor");
Vertex v3 = graph.NewVertex(instance);
v3.SetProperty(instanceName, "age");
session.Commit();
}
using (var session = new SessionNoServer(@"d:\graphtest2"))
{
//session.DefaultDatabaseLocation().CompressPages = true;
DataCache.MaximumMemoryUse = 4000000000;
var sw = new Stopwatch();
sw.Start();
//int i = 0;
session.BeginRead();
var graph = Graph.Open(session);
//define schema Trace.Wri
VertexType[] vertexTypes = graph.FindVertexTypes();
//vertexTypes.Select(x => x.TypeName).PrintDump();
EdgeType[] edgeTypes = graph.FindEdgeTypes();
VertexType concept = vertexTypes.FirstOrDefault(x => x.TypeName == "Concept") ?? graph.NewVertexType("Concept");
PropertyType conceptName = concept.FindProperty("ConceptName");
Assert.IsNotNull(conceptName, "ConceptName");
PropertyType conceptSize = concept.FindProperty("ConceptSize");
Assert.IsNotNull(conceptSize, "ConceptSize");
PropertyType conceptFrequency = concept.FindProperty("ConceptFrequency");
//PropertyType similarity = concept.NewProperty("Similarity", VelocityGraph.DataType.Double,
// VelocityGraph.PropertyKind.NotIndexed);
PropertyType vagueness = concept.FindProperty("Vagueness");//, DataType.Double,PropertyKind.NotIndexed);
VertexType instance = vertexTypes.FirstOrDefault(x => x.TypeName == "Instance");
PropertyType instanceSize = instance.FindProperty("InstanceSize");
PropertyType instanceName = instance.FindProperty("InstanceName");
PropertyType instanceFrequency = instance.FindProperty("InstanceFrequency");
//VertexType attributes = graph.NewVertexType("Attribute");
////EdgeType hasAttirbute = attributes.edgattributes.NewHeadToTailEdge(nameScorePair,);
//EdgeType cooccursWith = graph.NewEdgeType("CooccursWith", true, instance, instance);
//PropertyType coocurrenceFrequency = namedScore.NewProperty("IntScore", DataType.Integer,
// PropertyKind.NotIndexed);
//VertexType synonym = graph.NewVertexType("Synonym", namedScore);
//PropertyType synonymScore = synonym.NewProperty("Score", DataType.Integer, PropertyKind.NotIndexed);
//EdgeType hasSynonym = graph.NewEdgeType("HasSynonym", true, synonym, instance);
EdgeType isA = edgeTypes.FirstOrDefault(x => x.TypeName == "IsA");
PropertyType frequency = isA.FindProperty("frequency");
PropertyType popularity = isA.FindProperty("popularity");
PropertyType ZipfSlope = isA.FindProperty("zipf_slope");
PropertyType ZipfPearson = isA.FindProperty("zipf_pearson");
EdgeType cooccurence = graph.FindEdgeType("Coocurrence");
//LRVertex vx1 = graph.NewVertex(instance);
//vx1.SetProperty("Name", "bla");
//LRVertex vx2 = graph.NewVertex(instance);
//vx2.SetProperty("bla", "name");
//LREdge edge = graph.NewEdge(cooccurence, vx1, vx2);
Assert.IsNotNull(conceptName);
Vertex f = graph.FindVertex(conceptName, "factor");
var inst = graph.FindVertex(instanceName, "age");
Assert.IsNotNull(f);
Assert.IsNotNull(inst);
}
//if (instanceVertex == null)
}
public void ingestData()
{
if (Directory.Exists(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir)))
{
Directory.Delete(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir), true); // remove systemDir from prior runs and all its databases.
}
Directory.CreateDirectory(Path.Combine(SessionBase.BaseDatabasePath, s_systemDir));
using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No))
{
session.BeginUpdate();
session.DefaultDatabaseLocation().CompressPages = PageInfo.compressionKind.LZ4;
Graph g = new Graph(session);
// SCHEMA
VertexType userType = g.NewVertexType("User");
EdgeType friendEdgeType = g.NewEdgeType("Friend", true, userType, userType);
PropertyType countryProperty = userType.NewProperty("country", DataType.String, PropertyKind.NotIndexed);
PropertyType incomeProperty = userType.NewProperty("income", DataType.Long, PropertyKind.NotIndexed);
PropertyType friendshipStartProperty = friendEdgeType.NewProperty("start", DataType.DateTime, PropertyKind.NotIndexed);
// DATA
int lineNumber = 0;
long fiendsCt = 0;
int stop = (int)Math.Pow(2, 26); // make sure to create enough of these
for (int i = 1; i < stop; i++)
{
g.NewVertex(userType);
}
session.Commit();
session.BeginUpdate();
foreach (string line in File.ReadLines(s_inputData))
{
if (++lineNumber % 10000 == 0)
{
Console.WriteLine("Parsing user " + lineNumber + ", friends total: " + fiendsCt + " at " + DateTime.Now);
}
string[] fields = line.Split(' ');
Vertex aUser = null;
foreach (string s in fields)
{
if (s.Length > 0)
{
if (aUser == null)
{
aUser = new Vertex(g, userType, int.Parse(s));
}
else
{
++fiendsCt;
Vertex aFriend = new Vertex(g, userType, int.Parse(s));
if (fiendsCt % 2 == 0)
{
aFriend.SetProperty(countryProperty, "Sweden"); // just some random stuff
}
else
{
aFriend.SetProperty(incomeProperty, fiendsCt); // just some random stuff
}
Edge edge = friendEdgeType.NewEdge(aUser, aFriend);
if (fiendsCt % 2 == 0)
{
edge.SetProperty(friendshipStartProperty, DateTime.Now);
}
}
}
}
if (DataCache.MaximumMemoryUse <= 27000000000)
{
if (lineNumber >= 100000) // remove this condition if you have time to wait a long while...
{
break;
}
}
}
Console.WriteLine("Done importing " + lineNumber + " users with " + fiendsCt + " friends");
session.Commit();
}
}