/// <summary> /// This function spills a small-degree vertex, stores its edges into seperate documents /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size /// NOTE: This function will upload the vertex document /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param> /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param> private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, out string existEdgeDocId, out string newEdgeDocId) { Debug.Assert(vertexObject["_partition"] != null); Debug.Assert((string)vertexObject["id"] == (string)vertexObject["_partition"]); // NOTE: The VertexCache is not updated here bool outEdgeSeperated = vertexObject["_edge"] is JObject; bool inEdgeSeperated = vertexObject["_reverse_edge"] is JObject; if (inEdgeSeperated && outEdgeSeperated) { throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated"); } JArray targetEdgeArray; bool targetEdgeIsReverse; if (inEdgeSeperated) { targetEdgeArray = (JArray)vertexObject["_edge"]; targetEdgeIsReverse = false; } else if (outEdgeSeperated) { targetEdgeArray = (JArray)vertexObject["_reverse_edge"]; targetEdgeIsReverse = true; } else { JArray outEdgeArray = (JArray)vertexObject["_edge"]; JArray inEdgeArray = (JArray)vertexObject["_reverse_edge"]; targetEdgeIsReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length); targetEdgeArray = targetEdgeIsReverse ? inEdgeArray : outEdgeArray; } // Create a new edge-document to store the currently creating edge JObject newEdgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = targetEdgeIsReverse, ["_is_reverse"] = targetEdgeIsReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = new JArray(targetEdgeArray.Last), }; newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result; targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now // Create another new edge-document to store the existing edges. JObject existEdgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = targetEdgeIsReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = targetEdgeArray, }; existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result; // Update vertexObject to store the newly create edge-document & upload the vertexObject vertexObject[targetEdgeIsReverse ? "_reverse_edge" : "_edge"] = new JObject { ["_edges"] = new JArray { new JObject { ["id"] = existEdgeDocId, }, new JObject { ["id"] = newEdgeDocId, }, }, }; bool dummyTooLarge; UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge); }
/// <summary> /// This function spills a small-degree vertex, stores its edges into seperate documents /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size /// NOTE: This function will upload the vertex document /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="spillReverse"> /// Whether to spill the outgoing edges or incoming edges. /// If it's null, let this function decide. /// (This happens when no spilling threshold is set but the document size limit is reached) /// </param> /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param> /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param> private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, ref bool?spillReverse, out string existEdgeDocId, out string newEdgeDocId) { Debug.Assert(vertexObject[KW_DOC_PARTITION] != null); Debug.Assert((string)vertexObject[KW_DOC_ID] == (string)vertexObject[KW_DOC_PARTITION]); if (spillReverse == null) { // Let this function decide whether incoming/outgoing edges to spill Debug.Assert(!IsSpilledVertex(vertexObject, true) || !IsSpilledVertex(vertexObject, false)); } else { Debug.Assert(!IsSpilledVertex(vertexObject, spillReverse.Value)); } // NOTE: The VertexCache is not updated here bool outEdgeSeperated = IsSpilledVertex(vertexObject, false); bool inEdgeSeperated = IsSpilledVertex(vertexObject, true); if (inEdgeSeperated && outEdgeSeperated) { throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated"); } JArray targetEdgeArray; if (inEdgeSeperated) { targetEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE]; spillReverse = false; } else if (outEdgeSeperated) { targetEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE]; spillReverse = true; } else { JArray outEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE]; JArray inEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE]; spillReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length); targetEdgeArray = spillReverse.Value ? inEdgeArray : outEdgeArray; } // Create a new edge-document to store the currently creating edge JObject newEdgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION], [KW_EDGEDOC_ISREVERSE] = spillReverse.Value, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_EDGE] = new JArray(targetEdgeArray.Last), }; newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result; targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now // Create another new edge-document to store the existing edges. JObject existEdgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION], [KW_EDGEDOC_ISREVERSE] = spillReverse.Value, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_EDGE] = targetEdgeArray, }; existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result; // Update vertexObject to store the newly create edge-document & upload the vertexObject vertexObject[spillReverse.Value ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE] = new JArray { // Store the last spilled edge document only. //new JObject { // [KW_DOC_ID] = existEdgeDocId, //}, new JObject { [KW_DOC_ID] = newEdgeDocId, }, }; // Update the vertex document to indicate whether it's spilled if (spillReverse.Value) { Debug.Assert((bool)vertexObject[KW_VERTEX_REVEDGE_SPILLED] == false); vertexObject[KW_VERTEX_REVEDGE_SPILLED] = true; } else { Debug.Assert((bool)vertexObject[KW_VERTEX_EDGE_SPILLED] == false); vertexObject[KW_VERTEX_EDGE_SPILLED] = true; } bool dummyTooLarge; UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge); Debug.Assert(!dummyTooLarge); }
/// <summary> /// Insert edgeObject to one a vertex. /// NOTE: vertex-document and edge-document(s) are uploaded. /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing /// edges in VertexCache are updated (from null to the newly created edge-document's id) /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually, /// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an /// edge's property, but not adding a new edge. /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param> /// <param name="edgeObject"></param> /// <param name="isReverse"></param> /// <param name="newEdgeDocId"></param> private static void InsertEdgeObjectInternal( GraphViewConnection connection, JObject vertexObject, VertexField vertexField, JObject edgeObject, bool isReverse, out string newEdgeDocId) { bool tooLarge; JToken edgeContainer = vertexObject[isReverse ? "_reverse_edge" : "_edge"]; // JArray or JObject if (edgeContainer is JObject) { // Now it is a large-degree vertex, and contains at least 1 edge-document JArray edgeDocumentsArray = (JArray)edgeContainer["_edges"]; Debug.Assert(edgeDocumentsArray != null, "edgeDocuments != null"); Debug.Assert(edgeDocumentsArray.Count > 0, "edgeDocuments.Count > 0"); string lastEdgeDocId = (string)edgeDocumentsArray.Last["id"]; JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId); Debug.Assert(((string)edgeDocument["id"]).Equals(lastEdgeDocId), "((string)edgeDocument['id']).Equals(lastEdgeDocId)"); Debug.Assert((bool)edgeDocument["_is_reverse"] == isReverse, "(bool)edgeDocument['_is_reverse'] == isReverse"); Debug.Assert((string)edgeDocument["_vertex_id"] == (string)vertexObject["id"], "(string)edgeDocument['_vertex_id'] == (string)vertexObject['id']"); JArray edgesArray = (JArray)edgeDocument["_edge"]; Debug.Assert(edgesArray != null, "edgesArray != null"); Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0"); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large edgesArray.Add(edgeObject); tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); if (edgesArray.Count >= connection.EdgeSpillThreshold) { // The threshold is reached! tooLarge = true; } else { // The threshold is not reached edgesArray.Add(edgeObject); tooLarge = false; } } // If the edge-document is not too large (reach the threshold), try to // upload the edge into the document if (!tooLarge) { UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge); } if (tooLarge) { // The edge is too large to be filled into the last edge-document // or the threashold is reached: // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { ["id"] = GraphViewConnection.GenerateDocumentId(), ["_partition"] = vertexObject["_partition"], ["_is_reverse"] = isReverse, ["_vertex_id"] = (string)vertexObject["id"], ["_edge"] = new JArray(edgeObject) }; lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result; // Add the newly create edge-document to vertexObject & upload the vertexObject edgeDocumentsArray.Add(new JObject { ["id"] = lastEdgeDocId }); } newEdgeDocId = lastEdgeDocId; // Upload the vertex documention (at least, its _nextXxx is changed) bool dummyTooLarge; UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge); Debug.Assert(!dummyTooLarge); } else if (edgeContainer is JArray) { ((JArray)edgeContainer).Add(edgeObject); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); tooLarge = (((JArray)edgeContainer).Count >= connection.EdgeSpillThreshold); } if (!tooLarge) { UploadOne(connection, (string)vertexObject["id"], vertexObject, out tooLarge); } if (tooLarge) { string existEdgeDocId; SpillVertexEdgesToDocument(connection, vertexObject, out existEdgeDocId, out newEdgeDocId); // Update the in & out edges in vertex field if (isReverse) { Debug.Assert(vertexField.RevAdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null)); foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges) { edge.EdgeDocID = existEdgeDocId; } } else { Debug.Assert(vertexField.AdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null)); foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges) { edge.EdgeDocID = existEdgeDocId; } } } else { newEdgeDocId = null; } } else { throw new Exception($"BUG: edgeContainer should either be JObject or JArray, but now: {edgeContainer?.GetType()}"); } }
/// <summary> /// Insert edgeObject to one a vertex. /// NOTE: vertex-document and edge-document(s) are uploaded. /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing /// edges in VertexCache are updated (from null to the newly created edge-document's id) /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually, /// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an /// edge's property, but not adding a new edge. /// </summary> /// <param name="connection"></param> /// <param name="vertexObject"></param> /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param> /// <param name="edgeObject"></param> /// <param name="isReverse"></param> /// <param name="newEdgeDocId"></param> private static void InsertEdgeObjectInternal( GraphViewConnection connection, JObject vertexObject, VertexField vertexField, JObject edgeObject, bool isReverse, out string newEdgeDocId) { bool tooLarge; JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject bool isSpilled = IsSpilledVertex(vertexObject, isReverse); if (isSpilled) { // Now it is a large-degree vertex, and contains at least 1 edge-document JArray edgeDocumentsArray = edgeContainer; Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null"); Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1"); string lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID]; JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId); Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)"); Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse"); Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']"); JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE]; Debug.Assert(edgesArray != null, "edgesArray != null"); Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0"); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large edgesArray.Add(edgeObject); tooLarge = false; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); if (edgesArray.Count >= connection.EdgeSpillThreshold) { // The threshold is reached! tooLarge = true; } else { // The threshold is not reached edgesArray.Add(edgeObject); tooLarge = false; } } // If the edge-document is not too large (reach the threshold), try to // upload the edge into the document if (!tooLarge) { UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge); } if (tooLarge) { // The edge is too large to be filled into the last edge-document // or the threashold is reached: // Create a new edge-document to store the edge. JObject edgeDocObject = new JObject { [KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(), [KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION], [KW_EDGEDOC_ISREVERSE] = isReverse, [KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID], [KW_EDGEDOC_EDGE] = new JArray(edgeObject) }; lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result; //// Add the newly create edge-document to vertexObject & upload the vertexObject //edgeDocumentsArray.Add(new JObject { // [KW_DOC_ID] = lastEdgeDocId //}); // Replace the newly created edge-document to vertexObject Debug.Assert(edgeDocumentsArray.Count == 1); edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId; } newEdgeDocId = lastEdgeDocId; // Upload the vertex documention (at least, its _nextXxx is changed) bool dummyTooLarge; UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge); Debug.Assert(!dummyTooLarge); } else { // This vertex is not spilled bool?spillReverse; ((JArray)edgeContainer).Add(edgeObject); if (connection.EdgeSpillThreshold == 0) { // Don't spill an edge-document until it is too large tooLarge = false; spillReverse = null; } else { // Explicitly specified a threshold Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0"); tooLarge = (((JArray)edgeContainer).Count > connection.EdgeSpillThreshold); spillReverse = isReverse; } if (!tooLarge) { UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out tooLarge); } if (tooLarge) { string existEdgeDocId; // The vertex object is uploaded in SpillVertexEdgesToDocument SpillVertexEdgesToDocument(connection, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId); // Update the in & out edges in vertex field Debug.Assert(spillReverse != null); Debug.Assert(vertexField != null); if (spillReverse.Value) { foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges) { Debug.Assert(edge.EdgeDocID == null); edge.EdgeDocID = existEdgeDocId; } } else { foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges) { Debug.Assert(edge.EdgeDocID == null); edge.EdgeDocID = existEdgeDocId; } } } else { newEdgeDocId = null; } } }