// TODO: Batch upload for the DropEdge part
internal override RawRecord DataModify(RawRecord record)
{
string vertexId = record[this._nodeIdIndex].ToValue;
// Temporarily change
DropEdgeOperator dropEdgeOp = new DropEdgeOperator(null, this.Connection, 0, 1);
RawRecord temp = new RawRecord(2);
VertexField vertex = this.Connection.VertexCache.GetVertexField(vertexId);
// Save a copy of Edges _IDs & drop outgoing edges
List <long> outEdgeOffsets = vertex.AdjacencyList.AllEdges.Select(e => e.Offset).ToList();
foreach (long outEdgeOffset in outEdgeOffsets)
{
temp.fieldValues[0] = new StringField(vertexId);
temp.fieldValues[1] = new StringField(outEdgeOffset.ToString());
dropEdgeOp.DataModify(temp);
}
AdjacencyListField revAdjacencyListField = Connection.UseReverseEdges
? vertex.RevAdjacencyList
: EdgeDocumentHelper.GetReverseAdjacencyListOfVertex(Connection, vertexId);
// Save a copy of incoming Edges <srcVertexId, edgeOffsetInSrcVertex> & drop them
List <Tuple <string, long> > inEdges = revAdjacencyListField.AllEdges.Select(
e => new Tuple <string, long>(e.OutV, e.Offset)).ToList();
foreach (var inEdge in inEdges)
{
temp.fieldValues[0] = new StringField(inEdge.Item1); // srcVertexId
temp.fieldValues[1] = new StringField(inEdge.Item2.ToString()); // edgeOffsetInSrcVertex
dropEdgeOp.DataModify(temp);
}
// Delete the node-document!
#if DEBUG
JObject vertexObject = this.Connection.RetrieveDocumentById(vertexId);
Debug.Assert(vertexObject != null);
Debug.Assert(vertexObject["_edge"] is JArray);
Debug.Assert(((JArray)vertexObject["_edge"]).Count == 0);
Debug.Assert(vertexObject["_reverse_edge"] is JArray);
Debug.Assert(((JArray)vertexObject["_reverse_edge"]).Count == 0);
#endif
// NOTE: for vertex document, id = _partition
this.Connection.ReplaceOrDeleteDocumentAsync(vertexId, null, vertexId).Wait();
// Update VertexCache
this.Connection.VertexCache.TryRemoveVertexField(vertexId);
return(null);
}
internal override RawRecord DataModify(RawRecord record)
{
string srcId = record[this._srcIdIndex].ToValue;
long edgeOffset = long.Parse(record[this._edgeOffsetIndex].ToValue);
JObject srcEdgeObject;
string srcEdgeDocId, sinkEdgeDocId;
JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcId);
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, srcVertexObject, srcId, edgeOffset, false,
out srcEdgeObject, out srcEdgeDocId);
if (srcEdgeObject == null)
{
//TODO: Check is this condition alright?
return(null);
}
string sinkId = (string)srcEdgeObject["_sinkV"];
JObject sinkVertexObject;
if (!string.Equals(sinkId, srcId))
{
sinkVertexObject = this.Connection.RetrieveDocumentById(sinkId);
JObject sinkEdgeObject;
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, sinkVertexObject, srcId, edgeOffset, true,
out sinkEdgeObject, out sinkEdgeDocId);
}
else
{
sinkVertexObject = srcVertexObject; // NOTE: Must not use DeepClone() here!
sinkEdgeDocId = srcEdgeDocId;
}
Dictionary <string, Tuple <JObject, string> > uploadDocuments = new Dictionary <string, Tuple <JObject, string> >();
EdgeDocumentHelper.RemoveEdge(uploadDocuments, this.Connection, srcEdgeDocId, srcVertexObject, false, srcId, edgeOffset);
EdgeDocumentHelper.RemoveEdge(uploadDocuments, this.Connection, sinkEdgeDocId, sinkVertexObject, true, srcId, edgeOffset);
this.Connection.ReplaceOrDeleteDocumentsAsync(uploadDocuments).Wait();
VertexField srcVertexField = this.Connection.VertexCache.GetVertexField(srcId, srcVertexObject);
VertexField sinkVertexField = this.Connection.VertexCache.GetVertexField(sinkId, sinkVertexObject);
srcVertexField.AdjacencyList.RemoveEdgeField(srcId, edgeOffset);
sinkVertexField.RevAdjacencyList.RemoveEdgeField(srcId, edgeOffset);
return(null);
}
internal override RawRecord DataModify(RawRecord record)
{
long edgeOffset = long.Parse(record[this._edgeOffsetIndex].ToValue);
string srcVertexId = record[this._srcVertexIdIndex].ToValue;
JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcVertexId);
string outEdgeDocId;
JObject outEdgeObject;
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, srcVertexObject, srcVertexId, edgeOffset, false,
out outEdgeObject, out outEdgeDocId);
if (outEdgeObject == null)
{
// TODO: Is there something wrong?
Debug.WriteLine($"[UpdateEdgePropertiesOperator] The edge does not exist: vertexId = {srcVertexId}, edgeOffset = {edgeOffset}");
return(null);
}
string sinkVertexId = (string)outEdgeObject["_sinkV"];
JObject sinkVertexObject;
string inEdgeDocId;
JObject inEdgeObject;
if (sinkVertexId.Equals(srcVertexId))
{
sinkVertexObject = srcVertexObject; // NOTE: Must not use DeepClone() here!
}
else
{
sinkVertexObject = this.Connection.RetrieveDocumentById(sinkVertexId);
}
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, sinkVertexObject, srcVertexId, edgeOffset, true,
out inEdgeObject, out inEdgeDocId);
VertexField srcVertexField = this.Connection.VertexCache.GetVertexField(srcVertexId);
VertexField sinkVertexField = this.Connection.VertexCache.GetVertexField(sinkVertexId);
EdgeField outEdgeField = srcVertexField.AdjacencyList.GetEdgeField(srcVertexId, edgeOffset);
EdgeField inEdgeField = sinkVertexField.RevAdjacencyList.GetEdgeField(srcVertexId, edgeOffset);
// Drop all non-reserved properties
if (this.PropertiesToBeUpdated.Count == 1 &&
!this.PropertiesToBeUpdated[0].Item1.SingleQuoted &&
this.PropertiesToBeUpdated[0].Item1.Value.Equals("*", StringComparison.OrdinalIgnoreCase) &&
!this.PropertiesToBeUpdated[0].Item2.SingleQuoted &&
this.PropertiesToBeUpdated[0].Item2.Value.Equals("null", StringComparison.OrdinalIgnoreCase))
{
List <string> toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(outEdgeObject);
foreach (var propertyName in toBeDroppedProperties)
{
outEdgeField.EdgeProperties.Remove(propertyName);
}
toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(inEdgeObject);
foreach (var propertyName in toBeDroppedProperties)
{
inEdgeField.EdgeProperties.Remove(propertyName);
}
}
else
{
foreach (Tuple <WValueExpression, WValueExpression, int> tuple in this.PropertiesToBeUpdated)
{
WValueExpression keyExpression = tuple.Item1;
WValueExpression valueExpression = tuple.Item2;
if (this.Mode == UpdatePropertyMode.Set)
{
// Modify edgeObject (update the edge property)
JProperty updatedProperty = GraphViewJsonCommand.UpdateProperty(outEdgeObject, keyExpression, valueExpression);
// Update VertexCache
if (updatedProperty == null)
{
outEdgeField.EdgeProperties.Remove(keyExpression.Value);
}
else
{
outEdgeField.UpdateEdgeProperty(updatedProperty, outEdgeField);
}
// Modify edgeObject (update the edge property)
updatedProperty = GraphViewJsonCommand.UpdateProperty(inEdgeObject, keyExpression, valueExpression);
// Update VertexCache
if (updatedProperty == null)
{
inEdgeField.EdgeProperties.Remove(keyExpression.Value);
}
else
{
inEdgeField.UpdateEdgeProperty(updatedProperty, inEdgeField);
}
}
else
{
throw new NotImplementedException();
}
}
}
// Interact with DocDB to update the property
EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, srcVertexObject, outEdgeDocId, false, outEdgeObject);
EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, sinkVertexObject, inEdgeDocId, true, inEdgeObject);
//
// Drop edge property
//
if (this.PropertiesToBeUpdated.Any(t => t.Item2 == null))
{
return(null);
}
return(record);
}
internal override RawRecord DataModify(RawRecord record)
{
VertexField srcVertexField = _srcFunction.Evaluate(record) as VertexField;
VertexField sinkVertexField = _sinkFunction.Evaluate(record) as VertexField;
if (srcVertexField == null || sinkVertexField == null)
{
return(null);
}
string srcId = srcVertexField["id"].ToValue;
string sinkId = sinkVertexField["id"].ToValue;
//string srcJsonDocument = srcVertexField.JsonDocument;
//string sinkJsonDocument = sinkVertexField.JsonDocument;
JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcId);
JObject sinkVertexObject;
if (srcId.Equals(sinkId))
{
// MUST not use JObject.DeepClone() here!
sinkVertexObject = srcVertexObject;
}
else
{
sinkVertexObject = this.Connection.RetrieveDocumentById(sinkId);
}
//VertexField srcVertexField = (srcFieldObject as VertexField)
// ?? Connection.VertexCache.GetVertexField(srcId, srcVertexObject);
//VertexField sinkVertexField = (sinkFieldObject as VertexField)
// ?? Connection.VertexCache.GetVertexField(sinkId, sinkVertexObject);
//
// Interact with DocDB and add the edge
// - For a small-degree vertex (now filled into one document), insert the edge in-place
// - If the upload succeeds, done!
// - If the upload fails with size-limit-exceeded(SLE), put either incoming or outgoing edges into a seperate document
// - For a large-degree vertex (already spilled)
// - Update either incoming or outgoing edges in the seperate edge-document
// - If the upload fails with SLE, create a new document to store the edge, and update the vertex document
//
JObject outEdgeObject, inEdgeObject;
string outEdgeDocID, inEdgeDocID;
EdgeDocumentHelper.InsertEdgeAndUpload(this.Connection,
srcId, sinkId,
srcVertexField, sinkVertexField, this._edgeJsonObject,
srcVertexObject, sinkVertexObject,
out outEdgeObject, out outEdgeDocID,
out inEdgeObject, out inEdgeDocID);
//
// Update vertex's adjacency list and reverse adjacency list (in vertex field)
//
EdgeField outEdgeField = EdgeField.ConstructForwardEdgeField(srcId, srcVertexField["label"]?.ToValue, outEdgeDocID, outEdgeObject);
EdgeField inEdgeField = EdgeField.ConstructBackwardEdgeField(sinkId, sinkVertexField["label"]?.ToValue, inEdgeDocID, inEdgeObject);
srcVertexField.AdjacencyList.AddEdgeField(srcId, outEdgeField.Offset, outEdgeField);
sinkVertexField.RevAdjacencyList.AddEdgeField(srcId, inEdgeField.Offset, inEdgeField);
// Construct the newly added edge's RawRecord
RawRecord result = new RawRecord();
// source, sink, other, offset, *
result.Append(new StringField(srcId));
result.Append(new StringField(sinkId));
result.Append(new StringField(_otherVTag == 0 ? srcId : sinkId));
result.Append(new StringField(outEdgeObject["_offset"].ToString()));
result.Append(outEdgeField);
for (int i = GraphViewReservedProperties.ReservedEdgeProperties.Count; i < _edgeProperties.Count; i++)
{
FieldObject fieldValue = outEdgeField[_edgeProperties[i]];
result.Append(fieldValue);
}
return(result);
}
internal static string ToGraphSON(List <RawRecord> results, GraphViewConnection connection)
{
StringBuilder finalGraphSonResult = new StringBuilder("[");
HashSet <string> batchIdSet = new HashSet <string>();
Dictionary <int, VertexField> batchGraphSonDict = new Dictionary <int, VertexField>();
StringBuilder notBatchedGraphSonResult = new StringBuilder();
bool firstEntry = true;
foreach (RawRecord record in results)
{
if (firstEntry)
{
firstEntry = false;
}
else
{
notBatchedGraphSonResult.Append(", ");
}
FieldObject field = record[0];
VertexField vertexField = field as VertexField;
if (vertexField != null &&
(!vertexField.AdjacencyList.HasBeenFetched || !vertexField.RevAdjacencyList.HasBeenFetched))
{
string vertexId = vertexField[GraphViewKeywords.KW_DOC_ID].ToValue;
batchIdSet.Add(vertexId);
batchGraphSonDict.Add(notBatchedGraphSonResult.Length, vertexField);
continue;
}
notBatchedGraphSonResult.Append(field.ToGraphSON());
}
if (batchIdSet.Any())
{
EdgeDocumentHelper.ConstructSpilledAdjListsOrVirtualRevAdjListsOfVertices(connection, batchIdSet);
int startIndex = 0;
foreach (KeyValuePair <int, VertexField> kvp in batchGraphSonDict)
{
int insertedPosition = kvp.Key;
int length = insertedPosition - startIndex;
VertexField vertexField = kvp.Value;
finalGraphSonResult.Append(notBatchedGraphSonResult.ToString(startIndex, length));
finalGraphSonResult.Append(vertexField.ToGraphSON());
startIndex = insertedPosition;
}
finalGraphSonResult.Append(notBatchedGraphSonResult.ToString(startIndex,
notBatchedGraphSonResult.Length - startIndex));
}
else
{
finalGraphSonResult.Append(notBatchedGraphSonResult.ToString());
}
finalGraphSonResult.Append("]");
return(finalGraphSonResult.ToString());
}
public void Upload(GraphViewCommand command)
{
if (this.isAddE && this.isDropE)
{
return;
}
else if (this.isAddE)
{
string outEdgeDocId;
EdgeDocumentHelper.InsertEdgeObjectInternal(command, this.srcVertexField.VertexJObject, this.srcVertexField,
this.outEdgeField.EdgeJObject, false, out outEdgeDocId);
this.outEdgeField.EdgeDocID = outEdgeDocId;
if (this.UseReverseEdges)
{
string inEdgeDocId;
EdgeDocumentHelper.InsertEdgeObjectInternal(command, this.sinkVertexField.VertexJObject, this.sinkVertexField,
this.inEdgeField.EdgeJObject, true, out inEdgeDocId);
this.inEdgeField.EdgeDocID = inEdgeDocId;
}
}
else if (this.isDropE)
{
string edgeId = outEdgeField.EdgeId;
string srcId = outEdgeField.OutV;
string sinkId = outEdgeField.InV;
JObject outEdgeObject;
string outEdgeDocId;
EdgeDocumentHelper.FindEdgeBySourceAndEdgeId(
command, this.srcVertexField.VertexJObject, srcId, edgeId, false,
out outEdgeObject, out outEdgeDocId);
if (outEdgeObject == null)
{
// something wrong. the edge that we want to drop does not exist in db
return;
}
string inEdgeDocId = null;
if (this.UseReverseEdges)
{
if (!string.Equals(sinkId, srcId))
{
JObject dummySinkEdgeObject;
EdgeDocumentHelper.FindEdgeBySourceAndEdgeId(
command, this.sinkVertexField.VertexJObject, srcId, edgeId, true,
out dummySinkEdgeObject, out inEdgeDocId);
}
else
{
Debug.Assert(object.ReferenceEquals(this.sinkVertexField, this.srcVertexField));
Debug.Assert(this.sinkVertexField.VertexJObject == this.srcVertexField.VertexJObject);
inEdgeDocId = outEdgeDocId;
}
}
// <docId, <docJson, partition>>
Dictionary <string, Tuple <JObject, string> > uploadDocuments = new Dictionary <string, Tuple <JObject, string> >();
EdgeDocumentHelper.RemoveEdge(uploadDocuments, command, outEdgeDocId,
this.srcVertexField, false, srcId, edgeId);
if (this.UseReverseEdges)
{
EdgeDocumentHelper.RemoveEdge(uploadDocuments, command, inEdgeDocId,
this.sinkVertexField, true, srcId, edgeId);
}
command.Connection.ReplaceOrDeleteDocumentsAsync(uploadDocuments, command).Wait();
}
else if (this.deltaProperties.Count > 0)
{
string edgeId = outEdgeField.EdgeId;
string srcId = outEdgeField.OutV;
string sinkId = outEdgeField.InV;
JObject outEdgeObject;
string outEdgeDocId;
EdgeDocumentHelper.FindEdgeBySourceAndEdgeId(
command, this.srcVertexField.VertexJObject, srcId, edgeId, false,
out outEdgeObject, out outEdgeDocId);
if (outEdgeObject == null)
{
// something wrong. the edge that we want to update does not exist in db
return;
}
string inEdgeDocId = null;
if (this.UseReverseEdges)
{
if (!string.Equals(sinkId, srcId))
{
JObject inEdgeObject;
EdgeDocumentHelper.FindEdgeBySourceAndEdgeId(
command, this.sinkVertexField.VertexJObject, srcId, edgeId, true,
out inEdgeObject, out inEdgeDocId);
}
else
{
Debug.Assert(object.ReferenceEquals(this.sinkVertexField, this.srcVertexField));
Debug.Assert(this.sinkVertexField.VertexJObject == this.srcVertexField.VertexJObject);
inEdgeDocId = outEdgeDocId;
}
}
// Interact with DocDB to update the property
EdgeDocumentHelper.UpdateEdgeProperty(command, this.srcVertexField.VertexJObject, outEdgeDocId, false,
this.outEdgeField.EdgeJObject);
if (this.UseReverseEdges)
{
EdgeDocumentHelper.UpdateEdgeProperty(command, this.sinkVertexField.VertexJObject, inEdgeDocId, true,
this.inEdgeField.EdgeJObject);
}
}
}
/// <summary>
/// For every vertex in the vertexIdSet, retrieve their spilled edge documents to construct their forward or backward
/// adjacency list.
/// When connection.useReverseEdge is false, this method will retrieve all the forward edge documents whose sink
/// is the target vertex to build a virtual reverse adjacency list.
/// </summary>
/// <param name="command"></param>
/// <param name="edgeType"></param>
/// <param name="vertexIdSet"></param>
/// <param name="vertexPartitionKeySet"></param>
public static void ConstructLazyAdjacencyList(
GraphViewCommand command,
EdgeType edgeType,
HashSet <string> vertexIdSet,
HashSet <string> vertexPartitionKeySet)
{
if (!vertexIdSet.Any())
{
return;
}
const string ALIAS = "edgeDoc";
var queryBoth = new JsonQuery
{
EdgeAlias = ALIAS
};
queryBoth.AddSelectElement("*");
queryBoth.RawWhereClause = new WInPredicate(new WColumnReferenceExpression(ALIAS, KW_EDGEDOC_VERTEXID), vertexIdSet.ToList());
queryBoth.FlatProperties.Add(KW_EDGEDOC_VERTEXID);
if (vertexPartitionKeySet.Any())
{
// TODO: Refactor this.
queryBoth.WhereConjunction(new WInPredicate(new WValueExpression($"{ALIAS}{command.Connection.GetPartitionPathIndexer()}"), vertexPartitionKeySet.ToList()),
BooleanBinaryExpressionType.And);
}
if (edgeType == EdgeType.Outgoing)
{
queryBoth.WhereConjunction(new WBooleanComparisonExpression
{
ComparisonType = BooleanComparisonType.Equals,
FirstExpr = new WColumnReferenceExpression(ALIAS, KW_EDGEDOC_ISREVERSE),
SecondExpr = new WValueExpression("false")
}, BooleanBinaryExpressionType.And);
queryBoth.FlatProperties.Add(KW_EDGEDOC_ISREVERSE);
}
else if (edgeType == EdgeType.Incoming)
{
queryBoth.WhereConjunction(new WBooleanComparisonExpression
{
ComparisonType = BooleanComparisonType.Equals,
FirstExpr = new WColumnReferenceExpression(ALIAS, KW_EDGEDOC_ISREVERSE),
SecondExpr = new WValueExpression("true")
}, BooleanBinaryExpressionType.And);
queryBoth.FlatProperties.Add(KW_EDGEDOC_ISREVERSE);
}
// var qqq = queryBoth.ToJsonServerString();
List <JObject> edgeDocuments = command.Connection.CreateDatabasePortal().GetEdgeDocuments(queryBoth);
// Dictionary<vertexId, Dictionary<edgeDocumentId, edgeDocument>>
var edgeDict = new Dictionary <string, Dictionary <string, JObject> >();
foreach (JObject edgeDocument in edgeDocuments)
{
// Save edgeDocument's etag if necessary
command.VertexCache.SaveCurrentEtagNoOverride(edgeDocument);
}
EdgeDocumentHelper.FillEdgeDict(edgeDict, edgeDocuments);
//
// Use all edges whose sink is vertexId to construct a virtual reverse adjacency list of this vertex
//
if (!command.Connection.UseReverseEdges && edgeType.HasFlag(EdgeType.Incoming))
{
// TODO: old Version JsonQuery, delete it when you understand this query.
// string selectClause = $"{{" +
// $" \"{EdgeDocumentHelper.VirtualReverseEdgeObject}\": edge, " +
// $" \"{KW_EDGE_SRCV}\": doc.{KW_DOC_ID}, " +
// $" \"{KW_EDGE_SRCV_LABEL}\": doc.{KW_VERTEX_LABEL}," +
// (command.Connection.PartitionPath != null
// ? $" \"{KW_EDGE_SRCV_PARTITION}\": doc{command.Connection.GetPartitionPathIndexer()},"
// : "") +
// $" \"{KW_EDGEDOC_VERTEXID}\": doc.{KW_EDGEDOC_VERTEXID}," +
// $" \"{KW_EDGEDOC_VERTEX_LABEL}\": doc.{KW_EDGEDOC_VERTEX_LABEL}" +
// $"}} AS {EdgeDocumentHelper.VirtualReverseEdge}";
// string alise = "doc";
// string joinClause = $"JOIN edge IN doc.{DocumentDBKeywords.KW_VERTEX_EDGE}";
// string inClause = string.Join(", ", vertexIdSet.Select(vertexId => $"'{vertexId}'"));
// string whereSearchCondition = $"edge.{KW_EDGE_SINKV} IN ({inClause})";
const string NODE_ALISE_S = "node";
const string EDGE_ALISE_S = "edge";
var queryReversed = new JsonQuery
{
NodeAlias = NODE_ALISE_S,
EdgeAlias = EDGE_ALISE_S
};
// Construct select clause.
List <WPrimaryExpression> selectList = new List <WPrimaryExpression>
{
new WValueExpression($"{{ \"{EdgeDocumentHelper.VirtualReverseEdgeObject}\": "),
new WColumnReferenceExpression(EDGE_ALISE_S, "*"),
new WValueExpression($", \"{KW_EDGE_SRCV}\": "),
new WColumnReferenceExpression(NODE_ALISE_S, KW_DOC_ID),
new WValueExpression($", \"{KW_EDGE_SRCV_LABEL}\": "),
new WColumnReferenceExpression(NODE_ALISE_S, KW_VERTEX_LABEL),
new WValueExpression($", \"{KW_EDGEDOC_VERTEXID}\": "),
new WColumnReferenceExpression(NODE_ALISE_S, KW_EDGEDOC_VERTEXID),
new WValueExpression($", \"{KW_EDGEDOC_VERTEX_LABEL}\": "),
new WColumnReferenceExpression(NODE_ALISE_S, KW_EDGEDOC_VERTEX_LABEL),
};
if (command.Connection.PartitionPath != null)
{
selectList.Add(new WValueExpression($", \"{KW_EDGE_SRCV_PARTITION}\": "));
// TODO: hack operation, when meet columnName[0] = '[', the toDocDbString function will do something special.
selectList.Add(new WColumnReferenceExpression(NODE_ALISE_S, command.Connection.GetPartitionPathIndexer()));
}
selectList.Add(new WValueExpression("}"));
queryReversed.AddSelectElement(EdgeDocumentHelper.VirtualReverseEdge, selectList);
// Construct join clause
queryReversed.JoinDictionary.Add(EDGE_ALISE_S, $"{NODE_ALISE_S}.{DocumentDBKeywords.KW_VERTEX_EDGE}");
// construct where clause
queryReversed.RawWhereClause = new WInPredicate(
new WColumnReferenceExpression(EDGE_ALISE_S, KW_EDGE_SINKV),
new List <string>(vertexIdSet));
edgeDocuments = command.Connection.CreateDatabasePortal().GetEdgeDocuments(queryReversed);
List <JObject> virtualReverseEdgeDocuments = EdgeDocumentHelper.ConstructVirtualReverseEdgeDocuments(edgeDocuments);
EdgeDocumentHelper.FillEdgeDict(edgeDict, virtualReverseEdgeDocuments);
}
foreach (KeyValuePair <string, Dictionary <string, JObject> > pair in edgeDict)
{
string vertexId = pair.Key;
Dictionary <string, JObject> edgeDocDict = pair.Value; // contains both in & out edges
VertexField vertexField;
command.VertexCache.TryGetVertexField(vertexId, out vertexField);
vertexField.ConstructSpilledOrVirtualAdjacencyListField(edgeType, edgeDocDict);
vertexIdSet.Remove(vertexId);
}
foreach (string vertexId in vertexIdSet)
{
VertexField vertexField;
command.VertexCache.TryGetVertexField(vertexId, out vertexField);
vertexField.ConstructSpilledOrVirtualAdjacencyListField(edgeType, new Dictionary <string, JObject>());
}
}
/// <summary>
///
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexObject"></param>
/// <param name="edgeDocId"></param>
/// <param name="isReverse"></param>
/// <param name="newEdgeObject"></param>
public static void UpdateEdgeProperty(
GraphViewCommand command,
JObject vertexObject,
string edgeDocId, // Can be null
bool isReverse,
JObject newEdgeObject // With all metadata (including id, partition, srcV/sinkV, edgeId)
)
{
bool tooLarge;
string srcOrSinkVInEdgeObject = isReverse ? KW_EDGE_SRCV : KW_EDGE_SINKV;
if (edgeDocId == null)
{
JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE];
// Don't use JToken.Replace() here.
// Make sure the currently modified edge is the last child of edgeContainer, which
// garantees the newly created edge-document won't be too large.
//
// NOTE: The following line applies for both incomming and outgoing edge.
edgeContainer.Children <JObject>().First(
e => (string)e[KW_EDGE_ID] == (string)newEdgeObject[KW_EDGE_ID] &&
(string)e[srcOrSinkVInEdgeObject] == (string)newEdgeObject[srcOrSinkVInEdgeObject]
).Remove();
edgeContainer.Add(newEdgeObject);
UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out tooLarge);
if (tooLarge)
{
// Handle this situation: The updated edge is too large to be filled into the vertex-document
string existEdgeDocId, newEdgeDocId;
bool? spillReverse = null;
EdgeDocumentHelper.SpillVertexEdgesToDocument(command, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId);
}
}
else
{
// Large vertex
JObject edgeDocObject = command.Connection.RetrieveDocumentById(edgeDocId, command.Connection.GetDocumentPartition(vertexObject), command);
edgeDocObject[KW_EDGEDOC_EDGE].Children <JObject>().First(
e => (string)e[KW_EDGE_ID] == (string)newEdgeObject[KW_EDGE_ID] &&
(string)e[srcOrSinkVInEdgeObject] == (string)newEdgeObject[srcOrSinkVInEdgeObject]
).Remove();
((JArray)edgeDocObject[KW_EDGEDOC_EDGE]).Add(newEdgeObject);
UploadOne(command, edgeDocId, edgeDocObject, false, out tooLarge);
if (tooLarge)
{
if (command.Connection.EdgeSpillThreshold == 1)
{
throw new GraphViewException("The edge is too large to be stored in one document!");
}
// Handle this situation: The modified edge is too large to be filled into the original edge-document
// Remove the edgeObject added just now, and upload the original edge-document
((JArray)edgeDocObject[KW_EDGEDOC_EDGE]).Last.Remove();
UploadOne(command, edgeDocId, edgeDocObject, false, out tooLarge);
Debug.Assert(!tooLarge);
// Insert the edgeObject to one of the vertex's edge-documents
InsertEdgeObjectInternal(command, vertexObject, null, newEdgeObject, isReverse, out edgeDocId);
}
}
}
/// <summary>
/// Insert edgeObject to one a vertex.
/// NOTE: vertex-document and edge-document(s) are uploaded.
/// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
/// edges in VertexCache are updated (from null to the newly created edge-document's id)
/// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
/// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
/// edge's property, but not adding a new edge.
/// </summary>
/// <param name="command"></param>
/// <param name="vertexObject"></param>
/// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
/// <param name="edgeObject"></param>
/// <param name="isReverse"></param>
/// <param name="newEdgeDocId"></param>
internal static void InsertEdgeObjectInternal(
GraphViewCommand command,
JObject vertexObject,
VertexField vertexField,
JObject edgeObject,
bool isReverse,
out string newEdgeDocId)
{
bool tooLarge;
JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject
bool isSpilled = IsSpilledVertex(vertexObject, isReverse);
//
// This graph is compatible only, thus add an edge-document directly
//
if (command.Connection.GraphType != GraphType.GraphAPIOnly || command.Connection.EdgeSpillThreshold == 1)
{
Debug.Assert(command.Connection.EdgeSpillThreshold == 1);
// Create a new edge-document to store the edge.
JObject edgeDocObject = new JObject {
[KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(),
[KW_EDGEDOC_ISREVERSE] = isReverse,
[KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID],
[KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
[KW_EDGEDOC_EDGE] = new JArray(edgeObject),
[KW_EDGEDOC_IDENTIFIER] = (JValue)true,
};
if (command.Connection.PartitionPathTopLevel != null)
{
// This may be KW_DOC_PARTITION, maybe not
edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
}
// Upload the edge-document
bool dummyTooLarge;
UploadOne(command, (string)edgeDocObject[KW_DOC_ID], edgeDocObject, true, out dummyTooLarge);
Debug.Assert(!dummyTooLarge);
newEdgeDocId = (string)edgeDocObject[KW_DOC_ID];
return;
}
if (isSpilled)
{
// Now it is a large-degree vertex, and contains at least 1 edge-document
JArray edgeDocumentsArray = edgeContainer;
Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null");
Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1");
string lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID];
JObject edgeDocument = command.Connection.RetrieveDocumentById(lastEdgeDocId, vertexField.Partition, command);
Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)");
Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse");
Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']");
JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE];
Debug.Assert(edgesArray != null, "edgesArray != null");
Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");
if (command.Connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
edgesArray.Add(edgeObject);
tooLarge = false;
}
else
{
// Explicitly specified a threshold
Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
if (edgesArray.Count >= command.Connection.EdgeSpillThreshold)
{
// The threshold is reached!
tooLarge = true;
}
else
{
// The threshold is not reached
edgesArray.Add(edgeObject);
tooLarge = false;
}
}
// If the edge-document is not too large (reach the threshold), try to
// upload the edge into the document
if (!tooLarge)
{
UploadOne(command, lastEdgeDocId, edgeDocument, false, out tooLarge);
}
if (tooLarge)
{
// The edge is too large to be filled into the last edge-document
// or the threashold is reached:
// Create a new edge-document to store the edge.
JObject edgeDocObject = new JObject {
[KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(),
[KW_EDGEDOC_ISREVERSE] = isReverse,
[KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID],
[KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
[KW_EDGEDOC_EDGE] = new JArray(edgeObject),
[KW_EDGEDOC_IDENTIFIER] = (JValue)true,
};
if (command.Connection.PartitionPathTopLevel != null)
{
// This may be KW_DOC_PARTITION, maybe not
edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
}
lastEdgeDocId = command.Connection.CreateDocumentAsync(edgeDocObject, command).Result;
// Replace the newly created edge-document to vertexObject
Debug.Assert(edgeDocumentsArray.Count == 1);
edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId;
}
newEdgeDocId = lastEdgeDocId;
// Upload the vertex documention (at least, its _nextXxx is changed)
bool dummyTooLarge;
UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out dummyTooLarge);
Debug.Assert(!dummyTooLarge);
}
else
{
// This vertex is not spilled
bool?spillReverse;
((JArray)edgeContainer).Add(edgeObject);
if (command.Connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
tooLarge = false;
spillReverse = null;
}
else
{
// Explicitly specified a threshold
Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
tooLarge = (((JArray)edgeContainer).Count > command.Connection.EdgeSpillThreshold);
spillReverse = isReverse;
}
if (!tooLarge)
{
UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out tooLarge);
}
if (tooLarge)
{
string existEdgeDocId;
// The vertex object is uploaded in SpillVertexEdgesToDocument
EdgeDocumentHelper.SpillVertexEdgesToDocument(command, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId);
// the edges are spilled into two ducuments.
// one stores old edges(docId = existEdgeDocId), the other one stores the new edge.
// Because the new edge is not in the vertexCache, hence we can set all edges' docId as existEdgeDocId
Debug.Assert(spillReverse != null);
Debug.Assert(vertexField != null);
if (spillReverse.Value)
{
vertexField.RevAdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId);
}
else
{
vertexField.AdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId);
}
}
else
{
newEdgeDocId = null;
}
}
}
/// <summary>
/// For every vertex in the vertexIdSet, retrieve their spilled edge documents to construct their forward or backward
/// adjacency list.
/// When connection.useReverseEdge is false, this method will retrieve all the forward edge documents whose sink
/// is the target vertex to build a virtual reverse adjacency list.
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexIdSet"></param>
public static void ConstructSpilledAdjListsOrVirtualRevAdjListsOfVertices(GraphViewConnection connection,
HashSet <string> vertexIdSet)
{
if (!vertexIdSet.Any())
{
return;
}
string inClause = string.Join(", ", vertexIdSet.Select(vertexId => $"'{vertexId}'"));
string edgeDocumentsQuery =
$"SELECT * " +
$"FROM edgeDoc " +
$"WHERE edgeDoc.{KW_EDGEDOC_VERTEXID} IN ({inClause})";
List <dynamic> edgeDocuments = connection.ExecuteQuery(edgeDocumentsQuery).ToList();
// Dictionary<vertexId, Dictionary<edgeDocumentId, edgeDocument>>
Dictionary <string, Dictionary <string, JObject> > edgeDict =
new Dictionary <string, Dictionary <string, JObject> >();
foreach (JObject edgeDocument in edgeDocuments)
{
// Save edgeDocument's etag if necessary
connection.VertexCache.SaveCurrentEtagNoOverride(edgeDocument);
}
EdgeDocumentHelper.FillEdgeDict(edgeDict, edgeDocuments);
//
// Use all edges whose sink is vertexId to construct a virtual reverse adjacency list of this vertex
//
if (!connection.UseReverseEdges)
{
edgeDocumentsQuery =
$"SELECT {{" +
$" \"{EdgeDocumentHelper.VirtualReverseEdgeObject}\": edge, " +
$" \"{KW_EDGE_SRCV}\": doc.{KW_DOC_ID}, " +
$" \"{KW_EDGE_SRCV_LABEL}\": doc.{KW_EDGE_SRCV_LABEL}," +
$" \"{KW_EDGEDOC_VERTEXID}\": doc.{KW_EDGEDOC_VERTEXID}" +
$"}} AS {EdgeDocumentHelper.VirtualReverseEdge}\n" +
$"FROM doc\n" +
$"JOIN edge IN doc.{GraphViewKeywords.KW_VERTEX_EDGE}\n" +
$"WHERE edge.{KW_EDGE_SINKV} IN ({inClause})";
edgeDocuments = connection.ExecuteQuery(edgeDocumentsQuery).ToList();
Dictionary <string, string> labelOfSrcVertexOfSpilledEdges =
EdgeDocumentHelper.GetLabelOfSrcVertexOfSpilledEdges(connection, edgeDocuments);
List <JObject> virtualReverseEdgeDocuments =
EdgeDocumentHelper.ConstructVirtualReverseEdgeDocuments(edgeDocuments, labelOfSrcVertexOfSpilledEdges);
EdgeDocumentHelper.FillEdgeDict(edgeDict, virtualReverseEdgeDocuments.Cast <dynamic>().ToList());
}
foreach (KeyValuePair <string, Dictionary <string, JObject> > pair in edgeDict)
{
string vertexId = pair.Key;
Dictionary <string, JObject> edgeDocDict = pair.Value; // contains both in & out edges
VertexField vertexField;
connection.VertexCache.TryGetVertexField(vertexId, out vertexField);
vertexField.ConstructSpilledOrVirtualAdjacencyListField(edgeDocDict);
vertexIdSet.Remove(vertexId);
}
foreach (string vertexId in vertexIdSet)
{
VertexField vertexField;
connection.VertexCache.TryGetVertexField(vertexId, out vertexField);
vertexField.ConstructSpilledOrVirtualAdjacencyListField(new Dictionary <string, JObject>());
}
}
