/// <summary>
/// Convert to gremlin edge
/// </summary>
/// <param name="edge"></param>
/// <param name="serializer"></param>
/// <param name="outV"></param>
/// <param name="inV"></param>
/// <returns></returns>
public static GremlinEdge ToEdge <V1, E, V2>(this E edge, JsonSerializer serializer,
V1 outV, V2 inV)
{
outV.ToJElement(serializer, out var outId, out var outLabel, out var outPk);
inV.ToJElement(serializer, out var inId, out var inLabel, out var inPk);
var jEdge = edge.ToJElement(serializer, out var edgeId, out var edgeLabel,
out var edgePk);
var gedge = new GremlinEdge(edgeId, edgeLabel, outId, inId, outLabel,
inLabel, outPk, inPk);
foreach (var prop in jEdge)
{
switch (prop.Key.ToLowerInvariant())
{
case DocumentProperties.IdKey:
case DocumentProperties.LabelKey:
case DocumentProperties.PartitionKey:
break;
default:
gedge.AddProperty(prop.Key, prop.Value);
break;
}
}
return(gedge);
}
/// <summary>
/// Generates the rating edges.
/// </summary>
/// <returns></returns>
public static IEnumerable <GremlinEdge> GenerateRatingEdges()
{
var lines =
File.ReadAllLines(
ConfigurationManager.AppSettings["dataPath"] + @"\ratings_small.csv");
var edges = new List <GremlinEdge>();
for (var index = 1; index < lines.Length; index++)
{
var line = lines[index];
var data = line.Split(',');
var e = new GremlinEdge(Guid.NewGuid().ToString(),
"rated",
data[0],
data[1],
"user",
"movie",
data[0],
data[1]);
e.AddProperty("rating", data[2]);
e.AddProperty("timestamp", long.Parse(data[3]));
edges.Add(e);
}
return(edges);
}
private async Task RunBulkImportEdgesAsync(
GraphBulkExecutor graphbulkExecutor,
EdgeSpec spec,
int fromCount,
int toCount)
{
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
BulkImportResponse response = null;
Random r = new Random();
for (int i = 0; i < fromCount; i++)
{
List <GremlinEdge> edges = new List <GremlinEdge>();
int edgeCount = r.Next(spec.maxCount - spec.minCount) + spec.minCount;
for (int j = 0; j < edgeCount; j++)
{
HashSet <int> neighbors = new HashSet <int>();
int destination = r.Next(toCount);
if (neighbors.Contains(destination))
{
j--;
continue;
}
neighbors.Add(destination);
string id = createEdgeId(i, spec.from, destination, spec.to);
GremlinEdge e = new GremlinEdge(
id, spec.label, createVertexId(spec.from, i), createVertexId(spec.to, destination),
spec.from, spec.to, createVertexId(spec.from, i), createVertexId(spec.to, destination));
for (int k = 0; k < spec.numberOfProperties; k++)
{
e.AddProperty("property_" + k, Guid.NewGuid().ToString());
}
edges.Add(e);
}
try
{
response = await graphbulkExecutor.BulkImportAsync(
edges,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
}
}
}
public static void InteractiveLogOns(InteractiveLogon _,
Dictionary <string, string> deviceObjectIdToDeviceId)
{
try
{
var gremlinEdges = new List <GremlinEdge>();
deviceObjectIdToDeviceId.TryGetValue(_.DeviceId, out var deviceId);
if (deviceId == null)
{
return;
}
var gremlinEdge = new GremlinEdge(
deviceId + _.UserId,
"HasSession",
deviceId,
_.UserId,
nameof(Computer),
nameof(User),
deviceId.GetHashCode(),
_.UserId.GetHashCode());
gremlinEdges.Add(gremlinEdge);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
/// <summary>
/// Generates the belongsToCollection edges.
/// </summary>
/// <returns></returns>
public static IEnumerable <GremlinEdge> GenerateBelongsToCollectionEdges()
{
// Read the file as one string.
var text =
File.ReadAllText(
ConfigurationManager.AppSettings["dataPath"] + @"\Edges\belongsToCollectionEdges.json");
// deserialize tuple list. First item in every tuple is movie ID and the second one is collection ID
var tuples = JsonConvert.DeserializeObject <List <Tuple <string, string> > >(text);
var edges = new List <GremlinEdge>();
foreach (var tuple in tuples)
{
var e = new GremlinEdge(Guid.NewGuid().ToString(),
"belongsToCollection",
tuple.Item1,
tuple.Item2,
"movie",
"collection",
tuple.Item1,
tuple.Item2);
edges.Add(e);
}
return(edges);
}
public static void GroupMembership(Group _,
List <GroupMember> groupMembers)
{
var gremlinVertices = new List <GremlinVertex>();
var gremlinEdges = new List <GremlinEdge>();
var vertex = new GremlinVertex(_.Id, nameof(Models.BloodHound.Group));
vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode());
vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty);
gremlinVertices.Add(vertex);
groupMembers.ForEach(member =>
{
var gremlinEdge = new GremlinEdge(
_.Id + member.Id,
"MemberOf",
member.Id,
_.Id,
member.MemberType,
nameof(Models.BloodHound.Group),
member.Id.GetHashCode(),
_.Id.GetHashCode());
gremlinEdges.Add(gremlinEdge);
});
CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
private static GremlinEdge ConvertToGremlinEdge(JObject doc)
{
GremlinEdge ge = null;
if ((bool)doc.GetValue("_isEdge") == true)
{
// Populating the source and destination partition key on the edges
string outVertexId = (string)doc.GetValue("_vertexId");
string invertexId = (string)doc.GetValue("_sink");
string edgeId = (string)doc.GetValue("id");
string edgeLabel = (string)doc.GetValue("label");
string outVertexLabel = (string)doc.GetValue("_vertexLabel");
string invertexLabel = (string)doc.GetValue("_sinkLabel");
object outVertexPartitionKey = idPKMapping[outVertexId];
object inVertexPartitionKey = idPKMapping[invertexId];
ge = new GremlinEdge(
edgeId: edgeId,
edgeLabel: edgeLabel,
outVertexId: outVertexId,
inVertexId: invertexId,
outVertexLabel: outVertexLabel,
inVertexLabel: invertexLabel,
outVertexPartitionKey: outVertexPartitionKey,
inVertexPartitionKey: inVertexPartitionKey);
}
return(ge);
}
public static IEnumerable <GremlinEdge> GenerateEdgesCustom(long count)
{
int counter = 0;
string dirFile = @"..\..\data\edges";
foreach (string fileName in Directory.GetFiles(dirFile))
{
Console.Write(fileName);
string jsonValue = System.IO.File.ReadAllText(fileName);
var RootObjects = JsonConvert.DeserializeObject <List <RootObject> >(jsonValue);
foreach (var rootObject in RootObjects)
{
GremlinEdge e = new GremlinEdge(
"e" + counter + rootObject.vertex1,
rootObject.edge,
rootObject.vertex1 + " " + rootObject.pk,
rootObject.vertex2 + " " + rootObject.sinkpk,
//rootObject.vertex1,
//rootObject.vertex2,
"vertex",
"vertex",
rootObject.pk,
rootObject.sinkpk);
e.AddProperty("duration", 0);
counter++;
yield return(e);
}
}
}
internal static GremlinEdge CreateGremlinEdge(string edgeLabel, string sourceId, string destinationId,
string sourceLabel, string destinationLabel, string edgeIdSuffix = null)
{
var edgeId = $"{sourceId} -> {destinationId}{edgeIdSuffix}";
var edge = new GremlinEdge(edgeId, edgeLabel, sourceId, destinationId,
sourceLabel, destinationLabel, Utils.CreatePartitionKey(sourceId), Utils.CreatePartitionKey(destinationId));
edge.AddProperty("model", "primary");
return(edge);
}
public static void DeviceOwners(Device _,
List <DirectoryObject> ownerList,
List <DirectoryRole> directoryRoles)
{
try
{
var gremlinVertices = new List <GremlinVertex>();
var gremlinEdges = new List <GremlinEdge>();
var deviceOwnerGroups =
directoryRoles.Where(__ =>
AzureActiveDirectoryHelper.DeviceOwnerGroupDisplayNames.Contains(__.DisplayName));
var vertex = new GremlinVertex(_.Id, nameof(Computer));
vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode());
vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty);
gremlinVertices.Add(vertex);
ownerList.ForEach(__ =>
{
var user = (User)__;
var gremlinEdge = new GremlinEdge(
user.Id + _.Id,
"AdminTo",
user.Id,
_.Id,
nameof(User),
nameof(Computer),
user.Id.GetHashCode(),
_.Id.GetHashCode());
gremlinEdges.Add(gremlinEdge);
});
deviceOwnerGroups.ForEach(directoryRole =>
{
var gremlinEdge = new GremlinEdge(
directoryRole.Id + _.Id,
"AdminTo",
directoryRole.Id,
_.Id,
nameof(DirectoryRole),
nameof(Computer),
directoryRole.Id.GetHashCode(),
_.Id.GetHashCode());
gremlinEdges.Add(gremlinEdge);
});
CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
catch (ClientException ex)
{
throw new Exception(ex.Message);
}
}
public static void AppOwnership(Application app,
List <DirectoryObject> owners, string applicationAdministratorRoleId = "534a2975-e5b9-4ea1-b7da-deaec0a7c0aa")
{
try
{
var gremlinVertices = new List <GremlinVertex>();
var gremlinEdges = new List <GremlinEdge>();
var vertex = new GremlinVertex(app.AppId, nameof(Models.BloodHound.Application));
vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, app.AppId.GetHashCode());
vertex.AddProperty(nameof(app.DisplayName), app.DisplayName?.ToUpper() ?? string.Empty);
gremlinVertices.Add(vertex);
owners.ForEach(owner =>
{
var gremlinEdge = new GremlinEdge(
owner.Id + app.AppId,
"Owner",
owner.Id,
app.AppId,
nameof(User),
nameof(Models.BloodHound.Application),
owner.Id.GetHashCode(),
app.AppId.GetHashCode());
gremlinEdges.Add(gremlinEdge);
});
var gremlinEdge2 = new GremlinEdge(
applicationAdministratorRoleId + app.AppId,
"Owner",
applicationAdministratorRoleId,
app.AppId,
nameof(DirectoryRole),
nameof(Models.BloodHound.Application),
applicationAdministratorRoleId.GetHashCode(),
app.AppId.GetHashCode()
);
gremlinEdges.Add(gremlinEdge2);
CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
/// <summary>
/// Generates the actedIn and cast edges.
/// </summary>
/// <returns></returns>
public static IEnumerable <GremlinEdge> GenerateActedInAndCastEdges()
{
// Read the file as one string.
var text =
File.ReadAllText(
ConfigurationManager.AppSettings["dataPath"] + @"\Edges\castEdges.json");
// deserialize tuple list. First item in every tuple is movie ID and the second one is castMember object
var tuples = JsonConvert.DeserializeObject <List <Tuple <string, CastMember> > >(text);
var edges = new List <GremlinEdge>();
foreach (var tuple in tuples)
{
var e = new GremlinEdge(Guid.NewGuid().ToString(),
"actedIn",
tuple.Item2.id,
tuple.Item1,
"person",
"movie",
tuple.Item2.id,
tuple.Item1);
e.AddProperty("character", tuple.Item2.character);
e.AddProperty("creditId", tuple.Item2.credit_id);
e.AddProperty("order", tuple.Item2.order);
edges.Add(e);
// we add second edge going in opposite direction for better query performance
// when traversing the graph from movie vertices...
// and we label that edge as 'cast'
// the downside to that approach is information duplication together with bigger graph & index size
e = new GremlinEdge(Guid.NewGuid().ToString(),
"cast",
tuple.Item1,
tuple.Item2.id,
"movie",
"person",
tuple.Item1,
tuple.Item2.id);
e.AddProperty("character", tuple.Item2.character);
e.AddProperty("creditId", tuple.Item2.credit_id);
e.AddProperty("order", tuple.Item2.order);
edges.Add(e);
}
return(edges);
}
private GremlinEdge ToEdge(E e)
{
var outV = e.GetOutVertex();
var inV = e.GetInVertex();
var edge = new GremlinEdge(
e.GetId(),
e.GetLabel(),
e.GetOutVertexId(),
e.GetInVertexId(),
outV.Label,
inV.Label,
outV.PartitionKey,
inV.PartitionKey);
return(edge);
}
public static IEnumerable <GremlinEdge> GenerateEdges(long count)
{
for (long i = 0; i < count - 1; i++)
{
GremlinEdge e = new GremlinEdge(
"e" + i,
"knows",
i.ToString(),
(i + 1).ToString(),
"vertex",
"vertex",
i,
i + 1);
e.AddProperty("duration", i);
yield return(e);
}
}
/// <summary>
/// Generates the producedIn and productionCountry edges.
/// </summary>
/// <returns></returns>
public static IEnumerable <GremlinEdge> GenerateProducedInAndProductionCountryEdges()
{
// Read the file as one string.
var text =
File.ReadAllText(
ConfigurationManager.AppSettings["dataPath"] + @"\Edges\producedInEdges.json");
// deserialize tuple list. First item in every tuple is movie ID and the second one is country ID
var tuples = JsonConvert.DeserializeObject <List <Tuple <string, string> > >(text);
var edges = new List <GremlinEdge>();
foreach (var tuple in tuples)
{
var e = new GremlinEdge(Guid.NewGuid().ToString(),
"productionCountry",
tuple.Item2,
tuple.Item1,
"country",
"movie",
tuple.Item2,
tuple.Item1);
edges.Add(e);
// we add second edge going in opposite direction for better query performance
// when traversing the graph from movie vertices...
// the downside to that approach is information duplication together with bigger graph & index size
e = new GremlinEdge(Guid.NewGuid().ToString(),
"producedIn",
tuple.Item1,
tuple.Item2,
"movie",
"country",
tuple.Item1,
tuple.Item2);
edges.Add(e);
}
return(edges);
}
private static GremlinEdge ToCosmosDBEdge(Neo4jRelationship relationshipData)
{
var relationship = relationshipData.Relationship;
/* DO NOT use Neo4j's relationship.Id directly as edgeId
* Cosmos DB stores both vertices and edges in the same Container
* and if Neo4j Node and Relationship Ids are the same, documents will be overwritten.*/
var edge = new GremlinEdge
(
edgeId: $"edge_{relationship.Id}",
edgeLabel: relationship.Type,
outVertexId: relationship.StartNodeId.ToString(),
inVertexId: relationship.EndNodeId.ToString(),
outVertexLabel: relationshipData.SourceLabel,
inVertexLabel: relationshipData.SinkLabel,
outVertexPartitionKey: relationshipData.SourcePartitionKey,
inVertexPartitionKey: relationshipData.SinkPartitionKey
);
AddProperties(relationship.Properties, (n, v) => edge.AddProperty(n, v));
return(edge);
}
public static void GroupMembership <T>(Group _, List <T> groupMembers) where T : GroupMember
{
try
{
var gremlinVertices = new List <GremlinVertex>();
var gremlinEdges = new List <GremlinEdge>();
var vertex = new GremlinVertex(_.Id, nameof(AzureAdLateralMovement.Models.BloodHound.Group));
vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, _.Id.GetHashCode());
vertex.AddProperty(nameof(_.DisplayName), _.DisplayName?.ToUpper() ?? string.Empty);
gremlinVertices.Add(vertex);
groupMembers.ForEach(member =>
{
var gremlinEdge = new GremlinEdge(
_.Id + member.Id,
member is GroupOwner ? "Owner" : "MemberOf",
member.Id,
_.Id,
member.Type,
nameof(AzureAdLateralMovement.Models.BloodHound.Group),
member.Id.GetHashCode(),
_.Id.GetHashCode());
gremlinEdges.Add(gremlinEdge);
});
CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public static void Applications(
string appDisplayName,
string appId,
HashSet <string> permissionsSet,
string principalId)
{
try
{
var gremlinVertices = new List <GremlinVertex>();
var gremlinEdges = new List <GremlinEdge>();
var vertex = new GremlinVertex(appId, nameof(Application));
vertex.AddProperty(CosmosDbHelper.CollectionPartitionKey, appId.GetHashCode());
vertex.AddProperty(nameof(appDisplayName), appDisplayName?.ToUpper() ?? string.Empty);
vertex.AddProperty(nameof(permissionsSet), permissionsSet.ToDelimitedString(",") ?? string.Empty);
gremlinVertices.Add(vertex);
var outVertexId = principalId ?? "AccessToAllPrincipals";
var gremlinEdge = new GremlinEdge(
outVertexId + appId,
"Granted",
appId,
outVertexId,
nameof(Models.BloodHound.User),
nameof(Application),
appId.GetHashCode(),
outVertexId.GetHashCode());
gremlinEdge.AddProperty(nameof(permissionsSet), permissionsSet.ToDelimitedString(",") ?? string.Empty);
gremlinEdges.Add(gremlinEdge);
var mailPermissions = new List <string>
{
"Mail.Read", "Mail.ReadBasic", "Mail.ReadWrite", "Mail.Read.Shared", "Mail.ReadWrite.Shared",
"Mail.Send", "Mail.Send.Shared", "MailboxSettings.Read", "Mail.Read", "Mail.ReadWrite",
"Mail.Send", "MailboxSettings.Read", "MailboxSettings.ReadWrite"
};
if (permissionsSet.Overlaps(mailPermissions))
{
gremlinEdge = new GremlinEdge(
appId + "MailBoxes",
"CanManipulate",
appId,
"MailBoxes",
nameof(Application),
nameof(Application),
appId.GetHashCode(),
"MailBoxes".GetHashCode());
gremlinEdges.Add(gremlinEdge);
}
CosmosDbHelper.RunImportVerticesBlock.Post(gremlinVertices);
CosmosDbHelper.RunImportEdgesBlock.Post(gremlinEdges);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public static IEnumerable <GremlinVertex> ConvertGreenTripsToVertices(IEnumerable <GreenTrip> greenTrips, List <GremlinVertex> Veritices, List <GremlinEdge> edges)
{
foreach (GreenTrip trip in greenTrips)
{
string thisIndex = GreenTripVertexIndex++.ToString();
string greenTripKey = "greentrip." + thisIndex;
String greenTripPK = String.Format("{0}:{1}", trip.VendorID, thisIndex);
GremlinVertex v = new GremlinVertex(greenTripKey, "greentrip");
v.AddProperty(ConfigurationManager.AppSettings["CollectionPartitionKey"], greenTripPK);
v.AddProperty("vendorid", trip.VendorID);
v.AddProperty("lpep_pickup_datetime", trip.lpep_pickup_datetime);
v.AddProperty("PULocationID", trip.PULocationID);
v.AddProperty("DOLocationID", trip.DOLocationID);
//yield return v;
Veritices.Add(v);
//Add the edges here as will need a reference to the original objects
//pickup start
GremlinEdge e = new GremlinEdge(
"e.pickup." + thisIndex,
"has_pickup",
greenTripKey, //green trip
"location." + trip.PULocationID.ToString(), //location ID
"greentrip",
"location",
greenTripPK,
"locations");
edges.Add(e);
//now reverse -- not sure if needed (Dont think so unless want to reverse query)
//e = new GremlinEdge(
// "e.pickup_greentrip." + thisIndex,
// "pickup_greentrip",
// "location." + trip.PULocationID.ToString(), //location ID
// greenTripKey, //green trip
// "location",
// "greentrip",
// "locations",
// greenTripPK);
//edges.Add(e);
//dropoff
e = new GremlinEdge(
"e.dropoff." + thisIndex,
"has_dropoff",
greenTripKey, //green trip
"location." + trip.DOLocationID.ToString(), //location ID
"greentrip",
"location",
greenTripPK,
"locations");
edges.Add(e);
//now reverse -- not sure if needed (Dont think so unless want to reverse query)
//e = new GremlinEdge(
// "e.dropoff_greentrip." + thisIndex,
// "dropoff_greentrip",
// "location." + trip.DOLocationID.ToString(), //location ID
// greenTripKey, //green trip
// "location",
// "greentrip",
// "locations",
// greenTripPK);
//if (GreenTripVertexIndex == 2) break; //force just 10k records
}
return(null);
}
private async Task ProcessTasks()
{
List <String> documentsToBeImported = new List <string>();
FileReader reader = new FileReader(InputFolder);
documentsToBeImported = reader.LoadDocuments();
List <Tuple <String, String, String> > mapping = Utility.XmlToTuples(documentsToBeImported);
List <GremlinEdge> edges = new List <GremlinEdge>();
List <GremlinVertex> vertices = new List <GremlinVertex>();
foreach (var tuple in mapping)
{
string outvertex = tuple.Item1;
string invertex = tuple.Item2;
string type = tuple.Item3;
if (type == "E")
{
GremlinVertex vertex1 = new GremlinVertex(outvertex, "Emisor");
vertex1.AddProperty("pk", outvertex);
GremlinVertex vertex2 = new GremlinVertex(invertex, "CFDI");
vertex2.AddProperty("pk", invertex);
if (!vertices.Contains(vertex1))
{
vertices.Add(vertex1);
}
if (!vertices.Contains(vertex2))
{
vertices.Add(vertex2);
}
GremlinEdge edge = new GremlinEdge(outvertex + invertex, outvertex + invertex, outvertex, invertex, outvertex, invertex, outvertex, invertex);
if (!edges.Contains(edge))
{
edges.Add(edge);
}
}
else if (type == "R")
{
GremlinVertex vertex1 = new GremlinVertex(outvertex, "CFDI");
vertex1.AddProperty("pk", outvertex);
GremlinVertex vertex2 = new GremlinVertex(invertex, "Receptor");
vertex2.AddProperty("pk", invertex);
if (!vertices.Contains(vertex1))
{
vertices.Add(vertex1);
}
if (!vertices.Contains(vertex2))
{
vertices.Add(vertex2);
}
GremlinEdge edge = new GremlinEdge(outvertex + invertex, outvertex + invertex, outvertex, invertex, outvertex, invertex, outvertex, invertex);
if (!edges.Contains(edge))
{
edges.Add(edge);
}
}
}
await RunBulkImportAsync(edges, vertices);
}
