/// <summary> /// Convert to gremlin edge /// </summary> /// <param name="edge"></param> /// <param name="serializer"></param> /// <param name="outV"></param> /// <param name="inV"></param> /// <returns></returns> public static GremlinEdge ToEdge <V1, E, V2>(this E edge, JsonSerializer serializer, V1 outV, V2 inV) { outV.ToJElement(serializer, out var outId, out var outLabel, out var outPk); inV.ToJElement(serializer, out var inId, out var inLabel, out var inPk); var jEdge = edge.ToJElement(serializer, out var edgeId, out var edgeLabel, out var edgePk); var gedge = new GremlinEdge(edgeId, edgeLabel, outId, inId, outLabel, inLabel, outPk, inPk); foreach (var prop in jEdge) { switch (prop.Key.ToLowerInvariant()) { case DocumentProperties.IdKey: case DocumentProperties.LabelKey: case DocumentProperties.PartitionKey: break; default: gedge.AddProperty(prop.Key, prop.Value); break; } } return(gedge); }
/// <summary> /// Generates the rating edges. /// </summary> /// <returns></returns> public static IEnumerable <GremlinEdge> GenerateRatingEdges() { var lines = File.ReadAllLines( ConfigurationManager.AppSettings["dataPath"] + @"\ratings_small.csv"); var edges = new List <GremlinEdge>(); for (var index = 1; index < lines.Length; index++) { var line = lines[index]; var data = line.Split(','); var e = new GremlinEdge(Guid.NewGuid().ToString(), "rated", data[0], data[1], "user", "movie", data[0], data[1]); e.AddProperty("rating", data[2]); e.AddProperty("timestamp", long.Parse(data[3])); edges.Add(e); } return(edges); }
private async Task RunBulkImportEdgesAsync( GraphBulkExecutor graphbulkExecutor, EdgeSpec spec, int fromCount, int toCount) { var tokenSource = new CancellationTokenSource(); var token = tokenSource.Token; BulkImportResponse response = null; Random r = new Random(); for (int i = 0; i < fromCount; i++) { List <GremlinEdge> edges = new List <GremlinEdge>(); int edgeCount = r.Next(spec.maxCount - spec.minCount) + spec.minCount; for (int j = 0; j < edgeCount; j++) { HashSet <int> neighbors = new HashSet <int>(); int destination = r.Next(toCount); if (neighbors.Contains(destination)) { j--; continue; } neighbors.Add(destination); string id = createEdgeId(i, spec.from, destination, spec.to); GremlinEdge e = new GremlinEdge( id, spec.label, createVertexId(spec.from, i), createVertexId(spec.to, destination), spec.from, spec.to, createVertexId(spec.from, i), createVertexId(spec.to, destination)); for (int k = 0; k < spec.numberOfProperties; k++) { e.AddProperty("property_" + k, Guid.NewGuid().ToString()); } edges.Add(e); } try { response = await graphbulkExecutor.BulkImportAsync( edges, enableUpsert : true, disableAutomaticIdGeneration : true, maxConcurrencyPerPartitionKeyRange : null, maxInMemorySortingBatchSize : null, cancellationToken : token); } catch (DocumentClientException de) { Trace.TraceError("Document client exception: {0}", de); } catch (Exception e) { Trace.TraceError("Exception: {0}", e); } } }
public static void InteractiveLogOns(InteractiveLogon _, Dictionary <string, string> deviceObjectIdToDeviceId) { try { var gremlinEdges = new List <GremlinEdge>(); deviceObjectIdToDeviceId.TryGetValue(_.DeviceId, out var deviceId); if (deviceId == null) { return; } var gremlinEdge = new GremlinEdge( deviceId + _.UserId, "HasSession", deviceId, _.UserId, nameof(Computer), nameof(User), deviceId.GetHashCode(), _.UserId.GetHashCode()); gremlinEdges.Add(gremlinEdge); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); } catch (Exception e) { Console.WriteLine(e); throw; } }
/// <summary> /// Generates the belongsToCollection edges. /// </summary> /// <returns></returns> public static IEnumerable <GremlinEdge> GenerateBelongsToCollectionEdges() { // Read the file as one string. var text = File.ReadAllText( ConfigurationManager.AppSettings["dataPath"] + @"\Edges\belongsToCollectionEdges.json"); // deserialize tuple list. First item in every tuple is movie ID and the second one is collection ID var tuples = JsonConvert.DeserializeObject <List <Tuple <string, string> > >(text); var edges = new List <GremlinEdge>(); foreach (var tuple in tuples) { var e = new GremlinEdge(Guid.NewGuid().ToString(), "belongsToCollection", tuple.Item1, tuple.Item2, "movie", "collection", tuple.Item1, tuple.Item2); edges.Add(e); } return(edges); }
public static void GroupMembership(Group _, List <GroupMember> groupMembers) { var gremlinVertices = new List <GremlinVertex>(); var gremlinEdges = new List <GremlinEdge>(); var vertex = new GremlinVertex(_.Id, nameof(Models.BloodHound.Group)); vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode()); vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty); gremlinVertices.Add(vertex); groupMembers.ForEach(member => { var gremlinEdge = new GremlinEdge( _.Id + member.Id, "MemberOf", member.Id, _.Id, member.MemberType, nameof(Models.BloodHound.Group), member.Id.GetHashCode(), _.Id.GetHashCode()); gremlinEdges.Add(gremlinEdge); }); CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); }
private static GremlinEdge ConvertToGremlinEdge(JObject doc) { GremlinEdge ge = null; if ((bool)doc.GetValue("_isEdge") == true) { // Populating the source and destination partition key on the edges string outVertexId = (string)doc.GetValue("_vertexId"); string invertexId = (string)doc.GetValue("_sink"); string edgeId = (string)doc.GetValue("id"); string edgeLabel = (string)doc.GetValue("label"); string outVertexLabel = (string)doc.GetValue("_vertexLabel"); string invertexLabel = (string)doc.GetValue("_sinkLabel"); object outVertexPartitionKey = idPKMapping[outVertexId]; object inVertexPartitionKey = idPKMapping[invertexId]; ge = new GremlinEdge( edgeId: edgeId, edgeLabel: edgeLabel, outVertexId: outVertexId, inVertexId: invertexId, outVertexLabel: outVertexLabel, inVertexLabel: invertexLabel, outVertexPartitionKey: outVertexPartitionKey, inVertexPartitionKey: inVertexPartitionKey); } return(ge); }
public static IEnumerable <GremlinEdge> GenerateEdgesCustom(long count) { int counter = 0; string dirFile = @"..\..\data\edges"; foreach (string fileName in Directory.GetFiles(dirFile)) { Console.Write(fileName); string jsonValue = System.IO.File.ReadAllText(fileName); var RootObjects = JsonConvert.DeserializeObject <List <RootObject> >(jsonValue); foreach (var rootObject in RootObjects) { GremlinEdge e = new GremlinEdge( "e" + counter + rootObject.vertex1, rootObject.edge, rootObject.vertex1 + " " + rootObject.pk, rootObject.vertex2 + " " + rootObject.sinkpk, //rootObject.vertex1, //rootObject.vertex2, "vertex", "vertex", rootObject.pk, rootObject.sinkpk); e.AddProperty("duration", 0); counter++; yield return(e); } } }
internal static GremlinEdge CreateGremlinEdge(string edgeLabel, string sourceId, string destinationId, string sourceLabel, string destinationLabel, string edgeIdSuffix = null) { var edgeId = $"{sourceId} -> {destinationId}{edgeIdSuffix}"; var edge = new GremlinEdge(edgeId, edgeLabel, sourceId, destinationId, sourceLabel, destinationLabel, Utils.CreatePartitionKey(sourceId), Utils.CreatePartitionKey(destinationId)); edge.AddProperty("model", "primary"); return(edge); }
public static void DeviceOwners(Device _, List <DirectoryObject> ownerList, List <DirectoryRole> directoryRoles) { try { var gremlinVertices = new List <GremlinVertex>(); var gremlinEdges = new List <GremlinEdge>(); var deviceOwnerGroups = directoryRoles.Where(__ => AzureActiveDirectoryHelper.DeviceOwnerGroupDisplayNames.Contains(__.DisplayName)); var vertex = new GremlinVertex(_.Id, nameof(Computer)); vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode()); vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty); gremlinVertices.Add(vertex); ownerList.ForEach(__ => { var user = (User)__; var gremlinEdge = new GremlinEdge( user.Id + _.Id, "AdminTo", user.Id, _.Id, nameof(User), nameof(Computer), user.Id.GetHashCode(), _.Id.GetHashCode()); gremlinEdges.Add(gremlinEdge); }); deviceOwnerGroups.ForEach(directoryRole => { var gremlinEdge = new GremlinEdge( directoryRole.Id + _.Id, "AdminTo", directoryRole.Id, _.Id, nameof(DirectoryRole), nameof(Computer), directoryRole.Id.GetHashCode(), _.Id.GetHashCode()); gremlinEdges.Add(gremlinEdge); }); CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); } catch (ClientException ex) { throw new Exception(ex.Message); } }
public static void AppOwnership(Application app, List <DirectoryObject> owners, string applicationAdministratorRoleId = "534a2975-e5b9-4ea1-b7da-deaec0a7c0aa") { try { var gremlinVertices = new List <GremlinVertex>(); var gremlinEdges = new List <GremlinEdge>(); var vertex = new GremlinVertex(app.AppId, nameof(Models.BloodHound.Application)); vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, app.AppId.GetHashCode()); vertex.AddProperty(nameof(app.DisplayName), app.DisplayName?.ToUpper() ?? string.Empty); gremlinVertices.Add(vertex); owners.ForEach(owner => { var gremlinEdge = new GremlinEdge( owner.Id + app.AppId, "Owner", owner.Id, app.AppId, nameof(User), nameof(Models.BloodHound.Application), owner.Id.GetHashCode(), app.AppId.GetHashCode()); gremlinEdges.Add(gremlinEdge); }); var gremlinEdge2 = new GremlinEdge( applicationAdministratorRoleId + app.AppId, "Owner", applicationAdministratorRoleId, app.AppId, nameof(DirectoryRole), nameof(Models.BloodHound.Application), applicationAdministratorRoleId.GetHashCode(), app.AppId.GetHashCode() ); gremlinEdges.Add(gremlinEdge2); CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); } catch (Exception e) { Console.WriteLine(e); throw; } }
/// <summary> /// Generates the actedIn and cast edges. /// </summary> /// <returns></returns> public static IEnumerable <GremlinEdge> GenerateActedInAndCastEdges() { // Read the file as one string. var text = File.ReadAllText( ConfigurationManager.AppSettings["dataPath"] + @"\Edges\castEdges.json"); // deserialize tuple list. First item in every tuple is movie ID and the second one is castMember object var tuples = JsonConvert.DeserializeObject <List <Tuple <string, CastMember> > >(text); var edges = new List <GremlinEdge>(); foreach (var tuple in tuples) { var e = new GremlinEdge(Guid.NewGuid().ToString(), "actedIn", tuple.Item2.id, tuple.Item1, "person", "movie", tuple.Item2.id, tuple.Item1); e.AddProperty("character", tuple.Item2.character); e.AddProperty("creditId", tuple.Item2.credit_id); e.AddProperty("order", tuple.Item2.order); edges.Add(e); // we add second edge going in opposite direction for better query performance // when traversing the graph from movie vertices... // and we label that edge as 'cast' // the downside to that approach is information duplication together with bigger graph & index size e = new GremlinEdge(Guid.NewGuid().ToString(), "cast", tuple.Item1, tuple.Item2.id, "movie", "person", tuple.Item1, tuple.Item2.id); e.AddProperty("character", tuple.Item2.character); e.AddProperty("creditId", tuple.Item2.credit_id); e.AddProperty("order", tuple.Item2.order); edges.Add(e); } return(edges); }
private GremlinEdge ToEdge(E e) { var outV = e.GetOutVertex(); var inV = e.GetInVertex(); var edge = new GremlinEdge( e.GetId(), e.GetLabel(), e.GetOutVertexId(), e.GetInVertexId(), outV.Label, inV.Label, outV.PartitionKey, inV.PartitionKey); return(edge); }
public static IEnumerable <GremlinEdge> GenerateEdges(long count) { for (long i = 0; i < count - 1; i++) { GremlinEdge e = new GremlinEdge( "e" + i, "knows", i.ToString(), (i + 1).ToString(), "vertex", "vertex", i, i + 1); e.AddProperty("duration", i); yield return(e); } }
/// <summary> /// Generates the producedIn and productionCountry edges. /// </summary> /// <returns></returns> public static IEnumerable <GremlinEdge> GenerateProducedInAndProductionCountryEdges() { // Read the file as one string. var text = File.ReadAllText( ConfigurationManager.AppSettings["dataPath"] + @"\Edges\producedInEdges.json"); // deserialize tuple list. First item in every tuple is movie ID and the second one is country ID var tuples = JsonConvert.DeserializeObject <List <Tuple <string, string> > >(text); var edges = new List <GremlinEdge>(); foreach (var tuple in tuples) { var e = new GremlinEdge(Guid.NewGuid().ToString(), "productionCountry", tuple.Item2, tuple.Item1, "country", "movie", tuple.Item2, tuple.Item1); edges.Add(e); // we add second edge going in opposite direction for better query performance // when traversing the graph from movie vertices... // the downside to that approach is information duplication together with bigger graph & index size e = new GremlinEdge(Guid.NewGuid().ToString(), "producedIn", tuple.Item1, tuple.Item2, "movie", "country", tuple.Item1, tuple.Item2); edges.Add(e); } return(edges); }
private static GremlinEdge ToCosmosDBEdge(Neo4jRelationship relationshipData) { var relationship = relationshipData.Relationship; /* DO NOT use Neo4j's relationship.Id directly as edgeId * Cosmos DB stores both vertices and edges in the same Container * and if Neo4j Node and Relationship Ids are the same, documents will be overwritten.*/ var edge = new GremlinEdge ( edgeId: $"edge_{relationship.Id}", edgeLabel: relationship.Type, outVertexId: relationship.StartNodeId.ToString(), inVertexId: relationship.EndNodeId.ToString(), outVertexLabel: relationshipData.SourceLabel, inVertexLabel: relationshipData.SinkLabel, outVertexPartitionKey: relationshipData.SourcePartitionKey, inVertexPartitionKey: relationshipData.SinkPartitionKey ); AddProperties(relationship.Properties, (n, v) => edge.AddProperty(n, v)); return(edge); }
public static void GroupMembership <T>(Group _, List <T> groupMembers) where T : GroupMember { try { var gremlinVertices = new List <GremlinVertex>(); var gremlinEdges = new List <GremlinEdge>(); var vertex = new GremlinVertex(_.Id, nameof(AzureAdLateralMovement.Models.BloodHound.Group)); vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode()); vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty); gremlinVertices.Add(vertex); groupMembers.ForEach(member => { var gremlinEdge = new GremlinEdge( _.Id + member.Id, member is GroupOwner ? "Owner" : "MemberOf", member.Id, _.Id, member.Type, nameof(AzureAdLateralMovement.Models.BloodHound.Group), member.Id.GetHashCode(), _.Id.GetHashCode()); gremlinEdges.Add(gremlinEdge); }); CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); } catch (Exception e) { Console.WriteLine(e); throw; } }
public static void Applications( string appDisplayName, string appId, HashSet <string> permissionsSet, string principalId) { try { var gremlinVertices = new List <GremlinVertex>(); var gremlinEdges = new List <GremlinEdge>(); var vertex = new GremlinVertex(appId, nameof(Application)); vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, appId.GetHashCode()); vertex.AddProperty(nameof(appDisplayName), appDisplayName?.ToUpper() ?? string.Empty); vertex.AddProperty(nameof(permissionsSet), permissionsSet.ToDelimitedString(",") ?? string.Empty); gremlinVertices.Add(vertex); var outVertexId = principalId ?? "AccessToAllPrincipals"; var gremlinEdge = new GremlinEdge( outVertexId + appId, "Granted", appId, outVertexId, nameof(Models.BloodHound.User), nameof(Application), appId.GetHashCode(), outVertexId.GetHashCode()); gremlinEdge.AddProperty(nameof(permissionsSet), permissionsSet.ToDelimitedString(",") ?? string.Empty); gremlinEdges.Add(gremlinEdge); var mailPermissions = new List <string> { "Mail.Read", "Mail.ReadBasic", "Mail.ReadWrite", "Mail.Read.Shared", "Mail.ReadWrite.Shared", "Mail.Send", "Mail.Send.Shared", "MailboxSettings.Read", "Mail.Read", "Mail.ReadWrite", "Mail.Send", "MailboxSettings.Read", "MailboxSettings.ReadWrite" }; if (permissionsSet.Overlaps(mailPermissions)) { gremlinEdge = new GremlinEdge( appId + "MailBoxes", "CanManipulate", appId, "MailBoxes", nameof(Application), nameof(Application), appId.GetHashCode(), "MailBoxes".GetHashCode()); gremlinEdges.Add(gremlinEdge); } CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices); CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges); } catch (Exception e) { Console.WriteLine(e); throw; } }
public static IEnumerable <GremlinVertex> ConvertGreenTripsToVertices(IEnumerable <GreenTrip> greenTrips, List <GremlinVertex> Veritices, List <GremlinEdge> edges) { foreach (GreenTrip trip in greenTrips) { string thisIndex = GreenTripVertexIndex++.ToString(); string greenTripKey = "greentrip." + thisIndex; String greenTripPK = String.Format("{0}:{1}", trip.VendorID, thisIndex); GremlinVertex v = new GremlinVertex(greenTripKey, "greentrip"); v.AddProperty(ConfigurationManager.AppSettings["CollectionPartitionKey"], greenTripPK); v.AddProperty("vendorid", trip.VendorID); v.AddProperty("lpep_pickup_datetime", trip.lpep_pickup_datetime); v.AddProperty("PULocationID", trip.PULocationID); v.AddProperty("DOLocationID", trip.DOLocationID); //yield return v; Veritices.Add(v); //Add the edges here as will need a reference to the original objects //pickup start GremlinEdge e = new GremlinEdge( "e.pickup." + thisIndex, "has_pickup", greenTripKey, //green trip "location." + trip.PULocationID.ToString(), //location ID "greentrip", "location", greenTripPK, "locations"); edges.Add(e); //now reverse -- not sure if needed (Dont think so unless want to reverse query) //e = new GremlinEdge( // "e.pickup_greentrip." + thisIndex, // "pickup_greentrip", // "location." + trip.PULocationID.ToString(), //location ID // greenTripKey, //green trip // "location", // "greentrip", // "locations", // greenTripPK); //edges.Add(e); //dropoff e = new GremlinEdge( "e.dropoff." + thisIndex, "has_dropoff", greenTripKey, //green trip "location." + trip.DOLocationID.ToString(), //location ID "greentrip", "location", greenTripPK, "locations"); edges.Add(e); //now reverse -- not sure if needed (Dont think so unless want to reverse query) //e = new GremlinEdge( // "e.dropoff_greentrip." + thisIndex, // "dropoff_greentrip", // "location." + trip.DOLocationID.ToString(), //location ID // greenTripKey, //green trip // "location", // "greentrip", // "locations", // greenTripPK); //if (GreenTripVertexIndex == 2) break; //force just 10k records } return(null); }
private async Task ProcessTasks() { List <String> documentsToBeImported = new List <string>(); FileReader reader = new FileReader(InputFolder); documentsToBeImported = reader.LoadDocuments(); List <Tuple <String, String, String> > mapping = Utility.XmlToTuples(documentsToBeImported); List <GremlinEdge> edges = new List <GremlinEdge>(); List <GremlinVertex> vertices = new List <GremlinVertex>(); foreach (var tuple in mapping) { string outvertex = tuple.Item1; string invertex = tuple.Item2; string type = tuple.Item3; if (type == "E") { GremlinVertex vertex1 = new GremlinVertex(outvertex, "Emisor"); vertex1.AddProperty("pk", outvertex); GremlinVertex vertex2 = new GremlinVertex(invertex, "CFDI"); vertex2.AddProperty("pk", invertex); if (!vertices.Contains(vertex1)) { vertices.Add(vertex1); } if (!vertices.Contains(vertex2)) { vertices.Add(vertex2); } GremlinEdge edge = new GremlinEdge(outvertex + invertex, outvertex + invertex, outvertex, invertex, outvertex, invertex, outvertex, invertex); if (!edges.Contains(edge)) { edges.Add(edge); } } else if (type == "R") { GremlinVertex vertex1 = new GremlinVertex(outvertex, "CFDI"); vertex1.AddProperty("pk", outvertex); GremlinVertex vertex2 = new GremlinVertex(invertex, "Receptor"); vertex2.AddProperty("pk", invertex); if (!vertices.Contains(vertex1)) { vertices.Add(vertex1); } if (!vertices.Contains(vertex2)) { vertices.Add(vertex2); } GremlinEdge edge = new GremlinEdge(outvertex + invertex, outvertex + invertex, outvertex, invertex, outvertex, invertex, outvertex, invertex); if (!edges.Contains(edge)) { edges.Add(edge); } } } await RunBulkImportAsync(edges, vertices); }