/// <summary> /// Add an edge from one vertex (source) to another (sink) /// NOTE: Both the source and sink vertex are modified. /// NOTE: This function may upload the edge-document. /// NOTE: srcVertex and sinkVertex are updated and uploaded. /// </summary> /// <param name="connection"></param> /// <param name="srcId"></param> /// <param name="sinkId"></param> /// <param name="srcVertexField"></param> /// <param name="sinkVertexField"></param> /// <param name="edgeJsonObject"></param> /// <param name="srcVertexObject"></param> /// <param name="sinkVertexObject"></param> /// <param name="outEdgeObject"></param> /// <param name="outEdgeDocID"></param> /// <param name="inEdgeObject"></param> /// <param name="inEdgeDocID"></param> public static void InsertEdgeAndUpload( GraphViewConnection connection, string srcId, string sinkId, VertexField srcVertexField, VertexField sinkVertexField, JObject edgeJsonObject, JObject srcVertexObject, JObject sinkVertexObject, out JObject outEdgeObject, out string outEdgeDocID, out JObject inEdgeObject, out string inEdgeDocID) { long edgeOffset = (long)srcVertexObject["_nextEdgeOffset"]; srcVertexObject["_nextEdgeOffset"] = edgeOffset + 1; outEdgeObject = (JObject)edgeJsonObject.DeepClone(); inEdgeObject = (JObject)edgeJsonObject.DeepClone(); // Add "id" property to edgeObject if desired if (connection.GenerateEdgeId) { string guid = GraphViewConnection.GenerateDocumentId(); outEdgeObject["_edgeId"] = guid; inEdgeObject["_edgeId"] = guid; } string srcLabel = srcVertexObject["label"]?.ToString(); string sinkLabel = sinkVertexObject["label"]?.ToString(); GraphViewJsonCommand.UpdateEdgeMetaProperty(outEdgeObject, edgeOffset, false, sinkId, sinkLabel); GraphViewJsonCommand.UpdateEdgeMetaProperty(inEdgeObject, edgeOffset, true, srcId, srcLabel); InsertEdgeObjectInternal(connection, srcVertexObject, srcVertexField, outEdgeObject, false, out outEdgeDocID); // srcVertex uploaded InsertEdgeObjectInternal(connection, sinkVertexObject, sinkVertexField, inEdgeObject, true, out inEdgeDocID); // sinkVertex uploaded }
internal override RawRecord DataModify(RawRecord record) { JObject vertexObject = this._vertexDocument; string vertexId = GraphViewConnection.GenerateDocumentId(); Debug.Assert(vertexObject["id"] == null); Debug.Assert(vertexObject["_partition"] == null); vertexObject["id"] = vertexId; vertexObject["_partition"] = vertexId; this.Connection.CreateDocumentAsync(vertexObject).Wait(); VertexField vertexField = Connection.VertexCache.GetVertexField(vertexId, vertexObject); RawRecord result = new RawRecord(); foreach (string fieldName in _projectedFieldList) { FieldObject fieldValue = vertexField[fieldName]; result.Append(fieldValue); } return(result); }
private void UpdatePropertiesOfVertex(VertexField vertex) { JObject vertexDocument = this.Connection.RetrieveDocumentById(vertex.VertexId); foreach (WPropertyExpression property in this.updateProperties) { Debug.Assert(property.Value != null); VertexPropertyField vertexProperty; string name = property.Key.Value; // Construct single property JObject meta = new JObject(); foreach (KeyValuePair <WValueExpression, WValueExpression> pair in property.MetaProperties) { meta[pair.Key.Value] = pair.Value.ToJValue(); } JObject singleProperty = new JObject { ["_value"] = property.Value.ToJValue(), ["_propId"] = GraphViewConnection.GenerateDocumentId(), ["_meta"] = meta, }; // Set / Append to multiProperty JArray multiProperty; if (vertexDocument[name] == null) { multiProperty = new JArray(); vertexDocument[name] = multiProperty; } else { multiProperty = (JArray)vertexDocument[name]; } if (property.Cardinality == GremlinKeyword.PropertyCardinality.single) { multiProperty.Clear(); } multiProperty.Add(singleProperty); // Update vertex field bool existed = vertex.VertexProperties.TryGetValue(name, out vertexProperty); if (!existed) { vertexProperty = new VertexPropertyField(vertexDocument.Property(name), vertex); vertex.VertexProperties.Add(name, vertexProperty); } else { vertexProperty.Replace(vertexDocument.Property(name)); } } // Upload to DocDB this.Connection.ReplaceOrDeleteDocumentAsync(vertex.VertexId, vertexDocument, (string)vertexDocument["_partition"]).Wait(); }
public JObject ConstructNodeJsonDocument(string vertexLabel, List <WPropertyExpression> vertexProperties, out List <string> projectedFieldList) { Debug.Assert(vertexLabel != null); JObject vertexObject = new JObject { ["label"] = vertexLabel }; projectedFieldList = new List <string>(GraphViewReservedProperties.ReservedNodeProperties); foreach (WPropertyExpression vertexProperty in vertexProperties) { Debug.Assert(vertexProperty.Cardinality == GremlinKeyword.PropertyCardinality.list); if (!projectedFieldList.Contains(vertexProperty.Key.Value)) { projectedFieldList.Add(vertexProperty.Key.Value); } if (vertexProperty.Value.ToJValue() == null) { continue; } JObject meta = new JObject(); foreach (KeyValuePair <WValueExpression, WValueExpression> pair in vertexProperty.MetaProperties) { WValueExpression metaName = pair.Key; WValueExpression metaValue = pair.Value; meta[metaName.Value] = metaValue.ToJValue(); } string name = vertexProperty.Key.Value; JArray propArray = (JArray)vertexObject[name]; if (propArray == null) { propArray = new JArray(); vertexObject[name] = propArray; } propArray.Add(new JObject { ["_value"] = vertexProperty.Value.ToJValue(), ["_propId"] = GraphViewConnection.GenerateDocumentId(), ["_meta"] = meta, }); //GraphViewJsonCommand.AppendVertexSinglePropertyToVertex(vertexObject); } vertexObject["_edge"] = new JArray(); vertexObject["_reverse_edge"] = new JArray(); vertexObject["_nextEdgeOffset"] = 0; return(vertexObject); }
public JObject ConstructNodeJsonDocument(string vertexLabel, List <WPropertyExpression> vertexProperties) { Debug.Assert(vertexLabel != null); JObject vertexObject = new JObject { [KW_VERTEX_LABEL] = vertexLabel }; //projectedFieldList = new List<string>(GraphViewReservedProperties.InitialPopulateNodeProperties); //projectedFieldList.Add(GremlinKeyword.Label); foreach (WPropertyExpression vertexProperty in vertexProperties) { Debug.Assert(vertexProperty.Cardinality == GremlinKeyword.PropertyCardinality.List); //if (!projectedFieldList.Contains(vertexProperty.Key.Value)) // projectedFieldList.Add(vertexProperty.Key.Value); //if (vertexProperty.Value.ToJValue() == null) { // continue; //} JObject meta = new JObject(); foreach (KeyValuePair <WValueExpression, WValueExpression> pair in vertexProperty.MetaProperties) { WValueExpression metaName = pair.Key; WValueExpression metaValue = pair.Value; meta[metaName.Value] = metaValue.ToJValue(); } string name = vertexProperty.Key.Value; JArray propArray = (JArray)vertexObject[name]; if (propArray == null) { propArray = new JArray(); vertexObject[name] = propArray; } propArray.Add(new JObject { [KW_PROPERTY_VALUE] = vertexProperty.Value.ToJValue(), [KW_PROPERTY_ID] = GraphViewConnection.GenerateDocumentId(), [KW_PROPERTY_META] = meta, }); //GraphViewJsonCommand.AppendVertexSinglePropertyToVertex(vertexObject); } vertexObject[KW_VERTEX_EDGE] = new JArray(); vertexObject[KW_VERTEX_REV_EDGE] = new JArray(); vertexObject[KW_VERTEX_EDGE_SPILLED] = false; vertexObject[KW_VERTEX_REVEDGE_SPILLED] = false; //vertexObject[KW_VERTEX_NEXTOFFSET] = 0; return(vertexObject); }
public JObject ConstructNodeJsonDocument(out List <string> projectedFieldList) { JObject nodeJsonDocument = new JObject(); projectedFieldList = new List <string>(GraphViewReservedProperties.ReservedNodeProperties); for (var i = 0; i < Parameters.Count; i += 2) { var key = (Parameters[i] as WValueExpression).Value; //GraphViewJsonCommand.UpdateProperty(nodeJsonDocument, Parameters[i] as WValueExpression, // Parameters[i + 1] as WValueExpression); GraphViewJsonCommand.UpdateProperty(nodeJsonDocument, Parameters[i] as WValueExpression, Parameters[i + 1] as WValueExpression); string name = (Parameters[i] as WValueExpression).Value; JToken value = (Parameters[i + 1] as WValueExpression).ToJValue(); if (value != null) { if (VertexField.IsVertexMetaProperty(name)) { nodeJsonDocument[name] = value; } else { nodeJsonDocument[name] = new JArray { new JObject { ["_value"] = value, ["_propId"] = GraphViewConnection.GenerateDocumentId(), ["_meta"] = new JObject(), }, }; } } if (!projectedFieldList.Contains(key)) { projectedFieldList.Add(key); } } //nodeJsonDocument = GraphViewJsonCommand.insert_property(nodeJsonDocument, "[]", "_edge").ToString(); //nodeJsonDocument = GraphViewJsonCommand.insert_property(nodeJsonDocument, "[]", "_reverse_edge").ToString(); nodeJsonDocument["_edge"] = new JArray(); nodeJsonDocument["_reverse_edge"] = new JArray(); nodeJsonDocument["_nextEdgeOffset"] = 0; return(nodeJsonDocument); }
/// <summary> /// Add an edge from one vertex (source) to another (sink) /// NOTE: Both the source and sink vertex are modified. /// NOTE: This function may upload the edge-document. /// NOTE: srcVertex and sinkVertex are updated and uploaded. /// </summary> /// <param name="connection"></param> /// <param name="srcId"></param> /// <param name="sinkId"></param> /// <param name="srcVertexField"></param> /// <param name="sinkVertexField"></param> /// <param name="edgeJsonObject"></param> /// <param name="srcVertexObject"></param> /// <param name="sinkVertexObject"></param> /// <param name="outEdgeObject"></param> /// <param name="outEdgeDocID"></param> /// <param name="inEdgeObject"></param> /// <param name="inEdgeDocID"></param> public static void InsertEdgeAndUpload( GraphViewConnection connection, string srcId, string sinkId, VertexField srcVertexField, VertexField sinkVertexField, JObject edgeJsonObject, JObject srcVertexObject, JObject sinkVertexObject, out JObject outEdgeObject, out string outEdgeDocID, out JObject inEdgeObject, out string inEdgeDocID) { //long edgeOffset = (long)srcVertexObject[KW_VERTEX_NEXTOFFSET]; //srcVertexObject[KW_VERTEX_NEXTOFFSET] = edgeOffset + 1; outEdgeObject = (JObject)edgeJsonObject.DeepClone(); inEdgeObject = (JObject)edgeJsonObject.DeepClone(); // Add "id" property to edgeObject string edgeId = GraphViewConnection.GenerateDocumentId(); string srcLabel = srcVertexObject[KW_VERTEX_LABEL]?.ToString(); string sinkLabel = sinkVertexObject[KW_VERTEX_LABEL]?.ToString(); GraphViewJsonCommand.UpdateEdgeMetaProperty(outEdgeObject, edgeId, false, sinkId, sinkLabel); GraphViewJsonCommand.UpdateEdgeMetaProperty(inEdgeObject, edgeId, true, srcId, srcLabel); InsertEdgeObjectInternal(connection, srcVertexObject, srcVertexField, outEdgeObject, false, out outEdgeDocID); // srcVertex uploaded if (connection.UseReverseEdges) { InsertEdgeObjectInternal(connection, sinkVertexObject, sinkVertexField, inEdgeObject, true, out inEdgeDocID); // sinkVertex uploaded } else { inEdgeDocID = EdgeDocumentHelper.VirtualReverseEdgeDocId; } }
/// <summary> /// This function spills a small-degree vertex, stores its edges into seperate documents /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size /// NOTE: This function will upload the vertex document /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param> /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param> private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, out string existEdgeDocId, out string newEdgeDocId) { Debug.Assert(vertexObject["_partition"] != null); Debug.Assert((string)vertexObject["id"] == (string)vertexObject["_partition"]); // NOTE: The VertexCache is not updated here bool outEdgeSeperated = vertexObject["_edge"] is JObject; bool inEdgeSeperated = vertexObject["_reverse_edge"] is JObject; if (inEdgeSeperated && outEdgeSeperated) { throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated"); } JArray targetEdgeArray; bool targetEdgeIsReverse; if (inEdgeSeperated) { targetEdgeArray = (JArray)vertexObject["_edge"]; targetEdgeIsReverse = false; } else if (outEdgeSeperated) { targetEdgeArray = (JArray)vertexObject["_reverse_edge"]; targetEdgeIsReverse = true; } else { JArray outEdgeArray = (JArray)vertexObject["_edge"]; JArray inEdgeArray = (JArray)vertexObject["_reverse_edge"]; targetEdgeIsReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length); targetEdgeArray = targetEdgeIsReverse ? inEdgeArray : outEdgeArray; } // Create a new edge-document to store the currently creating edge JObject newEdgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = targetEdgeIsReverse, ["_is_reverse"] = targetEdgeIsReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = new JArray(targetEdgeArray.Last), }; newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result; targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now // Create another new edge-document to store the existing edges. JObject existEdgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = targetEdgeIsReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = targetEdgeArray, }; existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result; // Update vertexObject to store the newly create edge-document & upload the vertexObject vertexObject[targetEdgeIsReverse ? "_reverse_edge" : "_edge"] = new JObject { ["_edges"] = new JArray { new JObject { ["id"] = existEdgeDocId, }, new JObject { ["id"] = newEdgeDocId, }, }, }; bool dummyTooLarge; UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge); }
/// <summary> /// Insert edgeObject to one a vertex. /// NOTE: vertex-document and edge-document(s) are uploaded. /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing /// edges in VertexCache are updated (from null to the newly created edge-document's id) /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually, /// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an /// edge's property, but not adding a new edge. /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param> /// <param name="edgeObject"></param> /// <param name="isReverse"></param> /// <param name="newEdgeDocId"></param> private static void InsertEdgeObjectInternal( GraphViewConnection connection, JObject vertexObject, VertexField vertexField, JObject edgeObject, bool isReverse, out string newEdgeDocId) { bool tooLarge; JToken edgeContainer = vertexObject[isReverse ? "_reverse_edge" : "_edge"]; // JArray or JObject if (edgeContainer is JObject) { // Now it is a large-degree vertex, and contains at least 1 edge-document JArray edgeDocumentsArray = (JArray)edgeContainer["_edges"]; Debug.Assert(edgeDocumentsArray != null, "edgeDocuments != null"); Debug.Assert(edgeDocumentsArray.Count > 0, "edgeDocuments.Count > 0"); string lastEdgeDocId = (string)edgeDocumentsArray.Last["id"]; JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId); Debug.Assert(((string)edgeDocument["id"]).Equals(lastEdgeDocId), "((string)edgeDocument['id']).Equals(lastEdgeDocId)"); Debug.Assert((bool)edgeDocument["_is_reverse"] == isReverse, "(bool)edgeDocument['_is_reverse'] == isReverse"); Debug.Assert((string)edgeDocument["_vertex_id"] == (string)vertexObject["id"], "(string)edgeDocument['_vertex_id'] == (string)vertexObject['id']"); JArray edgesArray = (JArray)edgeDocument["_edge"]; Debug.Assert(edgesArray != null, "edgesArray != null"); Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0"); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large edgesArray.Add(edgeObject); tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); if (edgesArray.Count >= connection.EdgeSpillThreshold) { // The threshold is reached! tooLarge = true; } else { // The threshold is not reached edgesArray.Add(edgeObject); tooLarge = false; } } // If the edge-document is not too large (reach the threshold), try to // upload the edge into the document if (!tooLarge) { UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge); } if (tooLarge) { // The edge is too large to be filled into the last edge-document // or the threashold is reached: // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = isReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = new JArray(edgeObject) }; lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result; // Add the newly create edge-document to vertexObject & upload the vertexObject edgeDocumentsArray.Add(new JObject { ["id"] = lastEdgeDocId }); } newEdgeDocId = lastEdgeDocId; // Upload the vertex documention (at least, its _nextXxx is changed) bool dummyTooLarge; UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge); Debug.Assert(!dummyTooLarge); } else if (edgeContainer is JArray) { ((JArray)edgeContainer).Add(edgeObject); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); tooLarge = (((JArray)edgeContainer).Count >= connection.EdgeSpillThreshold); } if (!tooLarge) { UploadOne(connection, (string)vertexObject["id"], vertexObject, out tooLarge); } if (tooLarge) { string existEdgeDocId; SpillVertexEdgesToDocument(connection, vertexObject, out existEdgeDocId, out newEdgeDocId); // Update the in & out edges in vertex field if (isReverse) { Debug.Assert(vertexField.RevAdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null)); foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges) { edge.EdgeDocID = existEdgeDocId; } } else { Debug.Assert(vertexField.AdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null)); foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges) { edge.EdgeDocID = existEdgeDocId; } } } else { newEdgeDocId = null; } } else { throw new Exception($"BUG: edgeContainer should either be JObject or JArray, but now: {edgeContainer?.GetType()}"); } }
/// <summary> /// This function spills a small-degree vertex, stores its edges into seperate documents /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size /// NOTE: This function will upload the vertex document /// </summary> /// <param name="command"></param> /// <param name="vertexObject"></param> /// <param name="spillReverse"> /// Whether to spill the outgoing edges or incoming edges. /// If it's null, let this function decide. /// (This happens when no spilling threshold is set but the document size limit is reached) /// </param> /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param> /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param> private static void SpillVertexEdgesToDocument(GraphViewCommand command, JObject vertexObject, ref bool?spillReverse, out string existEdgeDocId, out string newEdgeDocId) { if (spillReverse == null) { // Let this function decide whether incoming/outgoing edges to spill Debug.Assert(!IsSpilledVertex(vertexObject, true) || !IsSpilledVertex(vertexObject, false)); } else { Debug.Assert(!IsSpilledVertex(vertexObject, spillReverse.Value)); } // NOTE: The VertexCache is not updated here bool outEdgeSeperated = IsSpilledVertex(vertexObject, false); bool inEdgeSeperated = IsSpilledVertex(vertexObject, true); if (inEdgeSeperated && outEdgeSeperated) { throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated"); } JArray targetEdgeArray; if (inEdgeSeperated) { targetEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE]; spillReverse = false; } else if (outEdgeSeperated) { targetEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE]; spillReverse = true; } else { JArray outEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE]; JArray inEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE]; spillReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length); targetEdgeArray = spillReverse.Value ? inEdgeArray : outEdgeArray; } // Create a new edge-document to store the currently creating edge JObject newEdgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_EDGEDOC_ISREVERSE] = spillReverse.Value, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL], [KW_EDGEDOC_EDGE] = new JArray(targetEdgeArray.Last), [KW_EDGEDOC_IDENTIFIER] = (JValue)true, }; if (command.Connection.PartitionPathTopLevel != null) { newEdgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel]; } newEdgeDocId = command.Connection.CreateDocumentAsync(newEdgeDocObject, command).Result; targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now // Create another new edge-document to store the existing edges. JObject existEdgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_EDGEDOC_ISREVERSE] = spillReverse.Value, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL], [KW_EDGEDOC_EDGE] = targetEdgeArray, [KW_EDGEDOC_IDENTIFIER] = (JValue)true, }; if (command.Connection.PartitionPathTopLevel != null) { existEdgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel]; } existEdgeDocId = command.Connection.CreateDocumentAsync(existEdgeDocObject, command).Result; // Update vertexObject to store the newly create edge-document & upload the vertexObject vertexObject[spillReverse.Value ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE] = new JArray { // Store the last spilled edge document only. new JObject { [KW_DOC_ID] = newEdgeDocId, }, }; // Update the vertex document to indicate whether it's spilled if (spillReverse.Value) { Debug.Assert((bool)vertexObject[KW_VERTEX_REVEDGE_SPILLED] == false); vertexObject[KW_VERTEX_REVEDGE_SPILLED] = true; } else { Debug.Assert((bool)vertexObject[KW_VERTEX_EDGE_SPILLED] == false); vertexObject[KW_VERTEX_EDGE_SPILLED] = true; } bool dummyTooLarge; UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out dummyTooLarge); Debug.Assert(!dummyTooLarge); }
/// <summary> /// Insert edgeObject to one a vertex. /// NOTE: vertex-document and edge-document(s) are uploaded. /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing /// edges in VertexCache are updated (from null to the newly created edge-document's id) /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually, /// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an /// edge's property, but not adding a new edge. /// </summary> /// <param name="command"></param> /// <param name="vertexObject"></param> /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param> /// <param name="edgeObject"></param> /// <param name="isReverse"></param> /// <param name="newEdgeDocId"></param> internal static void InsertEdgeObjectInternal( GraphViewCommand command, JObject vertexObject, VertexField vertexField, JObject edgeObject, bool isReverse, out string newEdgeDocId) { bool tooLarge; JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject bool isSpilled = IsSpilledVertex(vertexObject, isReverse); // // This graph is compatible only, thus add an edge-document directly // if (command.Connection.GraphType != GraphType.GraphAPIOnly || command.Connection.EdgeSpillThreshold == 1) { Debug.Assert(command.Connection.EdgeSpillThreshold == 1); // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_EDGEDOC_ISREVERSE] = isReverse, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL], [KW_EDGEDOC_EDGE] = new JArray(edgeObject), [KW_EDGEDOC_IDENTIFIER] = (JValue)true, }; if (command.Connection.PartitionPathTopLevel != null) { // This may be KW_DOC_PARTITION, maybe not edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel]; } // Upload the edge-document bool dummyTooLarge; UploadOne(command, (string)edgeDocObject[KW_DOC_ID], edgeDocObject, true, out dummyTooLarge); Debug.Assert(!dummyTooLarge); newEdgeDocId = (string)edgeDocObject[KW_DOC_ID]; return; } if (isSpilled) { // Now it is a large-degree vertex, and contains at least 1 edge-document JArray edgeDocumentsArray = edgeContainer; Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null"); Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1"); string lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID]; JObject edgeDocument = command.Connection.RetrieveDocumentById(lastEdgeDocId, vertexField.Partition, command); Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)"); Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse"); Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']"); JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE]; Debug.Assert(edgesArray != null, "edgesArray != null"); Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0"); if (command.Connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large edgesArray.Add(edgeObject); tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); if (edgesArray.Count >= command.Connection.EdgeSpillThreshold) { // The threshold is reached! tooLarge = true; } else { // The threshold is not reached edgesArray.Add(edgeObject); tooLarge = false; } } // If the edge-document is not too large (reach the threshold), try to // upload the edge into the document if (!tooLarge) { UploadOne(command, lastEdgeDocId, edgeDocument, false, out tooLarge); } if (tooLarge) { // The edge is too large to be filled into the last edge-document // or the threashold is reached: // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_EDGEDOC_ISREVERSE] = isReverse, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL], [KW_EDGEDOC_EDGE] = new JArray(edgeObject), [KW_EDGEDOC_IDENTIFIER] = (JValue)true, }; if (command.Connection.PartitionPathTopLevel != null) { // This may be KW_DOC_PARTITION, maybe not edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel]; } lastEdgeDocId = command.Connection.CreateDocumentAsync(edgeDocObject, command).Result; // Replace the newly created edge-document to vertexObject Debug.Assert(edgeDocumentsArray.Count == 1); edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId; } newEdgeDocId = lastEdgeDocId; // Upload the vertex documention (at least, its _nextXxx is changed) bool dummyTooLarge; UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out dummyTooLarge); Debug.Assert(!dummyTooLarge); } else { // This vertex is not spilled bool?spillReverse; ((JArray)edgeContainer).Add(edgeObject); if (command.Connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large tooLarge = false; spillReverse = null; } else { // Explicitly specified a threshold Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); tooLarge = (((JArray)edgeContainer).Count > command.Connection.EdgeSpillThreshold); spillReverse = isReverse; } if (!tooLarge) { UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out tooLarge); } if (tooLarge) { string existEdgeDocId; // The vertex object is uploaded in SpillVertexEdgesToDocument EdgeDocumentHelper.SpillVertexEdgesToDocument(command, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId); // the edges are spilled into two ducuments. // one stores old edges(docId = existEdgeDocId), the other one stores the new edge. // Because the new edge is not in the vertexCache, hence we can set all edges' docId as existEdgeDocId Debug.Assert(spillReverse != null); Debug.Assert(vertexField != null); if (spillReverse.Value) { vertexField.RevAdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId); } else { vertexField.AdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId); } } else { newEdgeDocId = null; } } }
/// <summary> /// Insert edgeObject to one a vertex. /// NOTE: vertex-document and edge-document(s) are uploaded. /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing /// edges in VertexCache are updated (from null to the newly created edge-document's id) /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually, /// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an /// edge's property, but not adding a new edge. /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param> /// <param name="edgeObject"></param> /// <param name="isReverse"></param> /// <param name="newEdgeDocId"></param> private static void InsertEdgeObjectInternal( GraphViewConnection connection, JObject vertexObject, VertexField vertexField, JObject edgeObject, bool isReverse, out string newEdgeDocId) { bool tooLarge; JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject bool isSpilled = IsSpilledVertex(vertexObject, isReverse); if (isSpilled) { // Now it is a large-degree vertex, and contains at least 1 edge-document JArray edgeDocumentsArray = edgeContainer; Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null"); Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1"); string lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID]; JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId); Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)"); Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse"); Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']"); JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE]; Debug.Assert(edgesArray != null, "edgesArray != null"); Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0"); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large edgesArray.Add(edgeObject); tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); if (edgesArray.Count >= connection.EdgeSpillThreshold) { // The threshold is reached! tooLarge = true; } else { // The threshold is not reached edgesArray.Add(edgeObject); tooLarge = false; } } // If the edge-document is not too large (reach the threshold), try to // upload the edge into the document if (!tooLarge) { UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge); } if (tooLarge) { // The edge is too large to be filled into the last edge-document // or the threashold is reached: // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION], [KW_EDGEDOC_ISREVERSE] = isReverse, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_EDGE] = new JArray(edgeObject) }; lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result; //// Add the newly create edge-document to vertexObject & upload the vertexObject //edgeDocumentsArray.Add(new JObject { // [KW_DOC_ID] = lastEdgeDocId //}); // Replace the newly created edge-document to vertexObject Debug.Assert(edgeDocumentsArray.Count == 1); edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId; } newEdgeDocId = lastEdgeDocId; // Upload the vertex documention (at least, its _nextXxx is changed) bool dummyTooLarge; UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge); Debug.Assert(!dummyTooLarge); } else { // This vertex is not spilled bool?spillReverse; ((JArray)edgeContainer).Add(edgeObject); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large tooLarge = false; spillReverse = null; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); tooLarge = (((JArray)edgeContainer).Count > connection.EdgeSpillThreshold); spillReverse = isReverse; } if (!tooLarge) { UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out tooLarge); } if (tooLarge) { string existEdgeDocId; // The vertex object is uploaded in SpillVertexEdgesToDocument SpillVertexEdgesToDocument(connection, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId); // Update the in & out edges in vertex field Debug.Assert(spillReverse != null); Debug.Assert(vertexField != null); if (spillReverse.Value) { foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges) { Debug.Assert(edge.EdgeDocID == null); edge.EdgeDocID = existEdgeDocId; } } else { foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges) { Debug.Assert(edge.EdgeDocID == null); edge.EdgeDocID = existEdgeDocId; } } } else { newEdgeDocId = null; } } }