/// <summary>
/// This function spills a small-degree vertex, stores its edges into seperate documents
/// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size
/// NOTE: This function will upload the vertex document
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexObject"></param>
/// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param>
/// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param>
private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, out string existEdgeDocId, out string newEdgeDocId)
{
Debug.Assert(vertexObject["_partition"] != null);
Debug.Assert((string)vertexObject["id"] == (string)vertexObject["_partition"]);
// NOTE: The VertexCache is not updated here
bool outEdgeSeperated = vertexObject["_edge"] is JObject;
bool inEdgeSeperated = vertexObject["_reverse_edge"] is JObject;
if (inEdgeSeperated && outEdgeSeperated)
{
throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated");
}
JArray targetEdgeArray;
bool targetEdgeIsReverse;
if (inEdgeSeperated)
{
targetEdgeArray = (JArray)vertexObject["_edge"];
targetEdgeIsReverse = false;
}
else if (outEdgeSeperated)
{
targetEdgeArray = (JArray)vertexObject["_reverse_edge"];
targetEdgeIsReverse = true;
}
else
{
JArray outEdgeArray = (JArray)vertexObject["_edge"];
JArray inEdgeArray = (JArray)vertexObject["_reverse_edge"];
targetEdgeIsReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length);
targetEdgeArray = targetEdgeIsReverse ? inEdgeArray : outEdgeArray;
}
// Create a new edge-document to store the currently creating edge
JObject newEdgeDocObject = new JObject {
["id"] = GraphViewConnection.GenerateDocumentId(),
["_partition"] = vertexObject["_partition"],
["_is_reverse"] = targetEdgeIsReverse,
["_is_reverse"] = targetEdgeIsReverse,
["_vertex_id"] = (string)vertexObject["id"],
["_edge"] = new JArray(targetEdgeArray.Last),
};
newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result;
targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now
// Create another new edge-document to store the existing edges.
JObject existEdgeDocObject = new JObject {
["id"] = GraphViewConnection.GenerateDocumentId(),
["_partition"] = vertexObject["_partition"],
["_is_reverse"] = targetEdgeIsReverse,
["_vertex_id"] = (string)vertexObject["id"],
["_edge"] = targetEdgeArray,
};
existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result;
// Update vertexObject to store the newly create edge-document & upload the vertexObject
vertexObject[targetEdgeIsReverse ? "_reverse_edge" : "_edge"] = new JObject {
["_edges"] = new JArray {
new JObject {
["id"] = existEdgeDocId,
},
new JObject {
["id"] = newEdgeDocId,
},
},
};
bool dummyTooLarge;
UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge);
}
/// <summary>
/// This function spills a small-degree vertex, stores its edges into seperate documents
/// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size
/// NOTE: This function will upload the vertex document
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexObject"></param>
/// <param name="spillReverse">
/// Whether to spill the outgoing edges or incoming edges.
/// If it's null, let this function decide.
/// (This happens when no spilling threshold is set but the document size limit is reached)
/// </param>
/// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param>
/// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param>
private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, ref bool?spillReverse, out string existEdgeDocId, out string newEdgeDocId)
{
Debug.Assert(vertexObject[KW_DOC_PARTITION] != null);
Debug.Assert((string)vertexObject[KW_DOC_ID] == (string)vertexObject[KW_DOC_PARTITION]);
if (spillReverse == null)
{
// Let this function decide whether incoming/outgoing edges to spill
Debug.Assert(!IsSpilledVertex(vertexObject, true) || !IsSpilledVertex(vertexObject, false));
}
else
{
Debug.Assert(!IsSpilledVertex(vertexObject, spillReverse.Value));
}
// NOTE: The VertexCache is not updated here
bool outEdgeSeperated = IsSpilledVertex(vertexObject, false);
bool inEdgeSeperated = IsSpilledVertex(vertexObject, true);
if (inEdgeSeperated && outEdgeSeperated)
{
throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated");
}
JArray targetEdgeArray;
if (inEdgeSeperated)
{
targetEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE];
spillReverse = false;
}
else if (outEdgeSeperated)
{
targetEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE];
spillReverse = true;
}
else
{
JArray outEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE];
JArray inEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE];
spillReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length);
targetEdgeArray = spillReverse.Value ? inEdgeArray : outEdgeArray;
}
// Create a new edge-document to store the currently creating edge
JObject newEdgeDocObject = new JObject {
[KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(),
[KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION],
[KW_EDGEDOC_ISREVERSE] = spillReverse.Value,
[KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID],
[KW_EDGEDOC_EDGE] = new JArray(targetEdgeArray.Last),
};
newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result;
targetEdgeArray.Last.Remove(); // Remove the currently create edge appended just now
// Create another new edge-document to store the existing edges.
JObject existEdgeDocObject = new JObject {
[KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(),
[KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION],
[KW_EDGEDOC_ISREVERSE] = spillReverse.Value,
[KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID],
[KW_EDGEDOC_EDGE] = targetEdgeArray,
};
existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result;
// Update vertexObject to store the newly create edge-document & upload the vertexObject
vertexObject[spillReverse.Value ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE] = new JArray {
// Store the last spilled edge document only.
//new JObject {
// [KW_DOC_ID] = existEdgeDocId,
//},
new JObject {
[KW_DOC_ID] = newEdgeDocId,
},
};
// Update the vertex document to indicate whether it's spilled
if (spillReverse.Value)
{
Debug.Assert((bool)vertexObject[KW_VERTEX_REVEDGE_SPILLED] == false);
vertexObject[KW_VERTEX_REVEDGE_SPILLED] = true;
}
else
{
Debug.Assert((bool)vertexObject[KW_VERTEX_EDGE_SPILLED] == false);
vertexObject[KW_VERTEX_EDGE_SPILLED] = true;
}
bool dummyTooLarge;
UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge);
Debug.Assert(!dummyTooLarge);
}
/// <summary>
/// Insert edgeObject to one a vertex.
/// NOTE: vertex-document and edge-document(s) are uploaded.
/// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
/// edges in VertexCache are updated (from null to the newly created edge-document's id)
/// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
/// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
/// edge's property, but not adding a new edge.
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexObject"></param>
/// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
/// <param name="edgeObject"></param>
/// <param name="isReverse"></param>
/// <param name="newEdgeDocId"></param>
private static void InsertEdgeObjectInternal(
GraphViewConnection connection,
JObject vertexObject,
VertexField vertexField,
JObject edgeObject,
bool isReverse,
out string newEdgeDocId)
{
bool tooLarge;
JToken edgeContainer = vertexObject[isReverse ? "_reverse_edge" : "_edge"]; // JArray or JObject
if (edgeContainer is JObject)
{
// Now it is a large-degree vertex, and contains at least 1 edge-document
JArray edgeDocumentsArray = (JArray)edgeContainer["_edges"];
Debug.Assert(edgeDocumentsArray != null, "edgeDocuments != null");
Debug.Assert(edgeDocumentsArray.Count > 0, "edgeDocuments.Count > 0");
string lastEdgeDocId = (string)edgeDocumentsArray.Last["id"];
JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId);
Debug.Assert(((string)edgeDocument["id"]).Equals(lastEdgeDocId), "((string)edgeDocument['id']).Equals(lastEdgeDocId)");
Debug.Assert((bool)edgeDocument["_is_reverse"] == isReverse, "(bool)edgeDocument['_is_reverse'] == isReverse");
Debug.Assert((string)edgeDocument["_vertex_id"] == (string)vertexObject["id"], "(string)edgeDocument['_vertex_id'] == (string)vertexObject['id']");
JArray edgesArray = (JArray)edgeDocument["_edge"];
Debug.Assert(edgesArray != null, "edgesArray != null");
Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");
if (connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
edgesArray.Add(edgeObject);
tooLarge = false;
}
else
{
// Explicitly specified a threshold
Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
if (edgesArray.Count >= connection.EdgeSpillThreshold)
{
// The threshold is reached!
tooLarge = true;
}
else
{
// The threshold is not reached
edgesArray.Add(edgeObject);
tooLarge = false;
}
}
// If the edge-document is not too large (reach the threshold), try to
// upload the edge into the document
if (!tooLarge)
{
UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge);
}
if (tooLarge)
{
// The edge is too large to be filled into the last edge-document
// or the threashold is reached:
// Create a new edge-document to store the edge.
JObject edgeDocObject = new JObject {
["id"] = GraphViewConnection.GenerateDocumentId(),
["_partition"] = vertexObject["_partition"],
["_is_reverse"] = isReverse,
["_vertex_id"] = (string)vertexObject["id"],
["_edge"] = new JArray(edgeObject)
};
lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result;
// Add the newly create edge-document to vertexObject & upload the vertexObject
edgeDocumentsArray.Add(new JObject {
["id"] = lastEdgeDocId
});
}
newEdgeDocId = lastEdgeDocId;
// Upload the vertex documention (at least, its _nextXxx is changed)
bool dummyTooLarge;
UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge);
Debug.Assert(!dummyTooLarge);
}
else if (edgeContainer is JArray)
{
((JArray)edgeContainer).Add(edgeObject);
if (connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
tooLarge = false;
}
else
{
// Explicitly specified a threshold
Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
tooLarge = (((JArray)edgeContainer).Count >= connection.EdgeSpillThreshold);
}
if (!tooLarge)
{
UploadOne(connection, (string)vertexObject["id"], vertexObject, out tooLarge);
}
if (tooLarge)
{
string existEdgeDocId;
SpillVertexEdgesToDocument(connection, vertexObject, out existEdgeDocId, out newEdgeDocId);
// Update the in & out edges in vertex field
if (isReverse)
{
Debug.Assert(vertexField.RevAdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null));
foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges)
{
edge.EdgeDocID = existEdgeDocId;
}
}
else
{
Debug.Assert(vertexField.AdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null));
foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges)
{
edge.EdgeDocID = existEdgeDocId;
}
}
}
else
{
newEdgeDocId = null;
}
}
else
{
throw new Exception($"BUG: edgeContainer should either be JObject or JArray, but now: {edgeContainer?.GetType()}");
}
}
/// <summary>
/// Insert edgeObject to one a vertex.
/// NOTE: vertex-document and edge-document(s) are uploaded.
/// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
/// edges in VertexCache are updated (from null to the newly created edge-document's id)
/// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
/// if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
/// edge's property, but not adding a new edge.
/// </summary>
/// <param name="connection"></param>
/// <param name="vertexObject"></param>
/// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
/// <param name="edgeObject"></param>
/// <param name="isReverse"></param>
/// <param name="newEdgeDocId"></param>
private static void InsertEdgeObjectInternal(
GraphViewConnection connection,
JObject vertexObject,
VertexField vertexField,
JObject edgeObject,
bool isReverse,
out string newEdgeDocId)
{
bool tooLarge;
JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject
bool isSpilled = IsSpilledVertex(vertexObject, isReverse);
if (isSpilled)
{
// Now it is a large-degree vertex, and contains at least 1 edge-document
JArray edgeDocumentsArray = edgeContainer;
Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null");
Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1");
string lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID];
JObject edgeDocument = connection.RetrieveDocumentById(lastEdgeDocId);
Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)");
Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse");
Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']");
JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE];
Debug.Assert(edgesArray != null, "edgesArray != null");
Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");
if (connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
edgesArray.Add(edgeObject);
tooLarge = false;
}
else
{
// Explicitly specified a threshold
Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
if (edgesArray.Count >= connection.EdgeSpillThreshold)
{
// The threshold is reached!
tooLarge = true;
}
else
{
// The threshold is not reached
edgesArray.Add(edgeObject);
tooLarge = false;
}
}
// If the edge-document is not too large (reach the threshold), try to
// upload the edge into the document
if (!tooLarge)
{
UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge);
}
if (tooLarge)
{
// The edge is too large to be filled into the last edge-document
// or the threashold is reached:
// Create a new edge-document to store the edge.
JObject edgeDocObject = new JObject {
[KW_DOC_ID] = GraphViewConnection.GenerateDocumentId(),
[KW_DOC_PARTITION] = vertexObject[KW_DOC_PARTITION],
[KW_EDGEDOC_ISREVERSE] = isReverse,
[KW_EDGEDOC_VERTEXID] = (string)vertexObject[KW_DOC_ID],
[KW_EDGEDOC_EDGE] = new JArray(edgeObject)
};
lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result;
//// Add the newly create edge-document to vertexObject & upload the vertexObject
//edgeDocumentsArray.Add(new JObject {
// [KW_DOC_ID] = lastEdgeDocId
//});
// Replace the newly created edge-document to vertexObject
Debug.Assert(edgeDocumentsArray.Count == 1);
edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId;
}
newEdgeDocId = lastEdgeDocId;
// Upload the vertex documention (at least, its _nextXxx is changed)
bool dummyTooLarge;
UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge);
Debug.Assert(!dummyTooLarge);
}
else
{
// This vertex is not spilled
bool?spillReverse;
((JArray)edgeContainer).Add(edgeObject);
if (connection.EdgeSpillThreshold == 0)
{
// Don't spill an edge-document until it is too large
tooLarge = false;
spillReverse = null;
}
else
{
// Explicitly specified a threshold
Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
tooLarge = (((JArray)edgeContainer).Count > connection.EdgeSpillThreshold);
spillReverse = isReverse;
}
if (!tooLarge)
{
UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out tooLarge);
}
if (tooLarge)
{
string existEdgeDocId;
// The vertex object is uploaded in SpillVertexEdgesToDocument
SpillVertexEdgesToDocument(connection, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId);
// Update the in & out edges in vertex field
Debug.Assert(spillReverse != null);
Debug.Assert(vertexField != null);
if (spillReverse.Value)
{
foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges)
{
Debug.Assert(edge.EdgeDocID == null);
edge.EdgeDocID = existEdgeDocId;
}
}
else
{
foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges)
{
Debug.Assert(edge.EdgeDocID == null);
edge.EdgeDocID = existEdgeDocId;
}
}
}
else
{
newEdgeDocId = null;
}
}
}