Ejemplo n.º 1
0
        /// <summary>
        /// Add an edge from one vertex (source) to another (sink)
        /// NOTE: Both the source and sink vertex are modified.
        /// NOTE: This function may upload the edge-document.
        /// NOTE: srcVertex and sinkVertex are updated and uploaded.
        /// </summary>
        /// <param name="connection"></param>
        /// <param name="srcId"></param>
        /// <param name="sinkId"></param>
        /// <param name="srcVertexField"></param>
        /// <param name="sinkVertexField"></param>
        /// <param name="edgeJsonObject"></param>
        /// <param name="srcVertexObject"></param>
        /// <param name="sinkVertexObject"></param>
        /// <param name="outEdgeObject"></param>
        /// <param name="outEdgeDocID"></param>
        /// <param name="inEdgeObject"></param>
        /// <param name="inEdgeDocID"></param>
        public static void InsertEdgeAndUpload(
            GraphViewConnection connection,
            string srcId, string sinkId,
            VertexField srcVertexField, VertexField sinkVertexField,
            JObject edgeJsonObject,
            JObject srcVertexObject, JObject sinkVertexObject,
            out JObject outEdgeObject, out string outEdgeDocID,
            out JObject inEdgeObject, out string inEdgeDocID)
        {
            long edgeOffset = (long)srcVertexObject["_nextEdgeOffset"];

            srcVertexObject["_nextEdgeOffset"] = edgeOffset + 1;

            outEdgeObject = (JObject)edgeJsonObject.DeepClone();
            inEdgeObject  = (JObject)edgeJsonObject.DeepClone();

            // Add "id" property to edgeObject if desired
            if (connection.GenerateEdgeId)
            {
                string guid = GraphViewConnection.GenerateDocumentId();
                outEdgeObject["_edgeId"] = guid;
                inEdgeObject["_edgeId"]  = guid;
            }

            string srcLabel  = srcVertexObject["label"]?.ToString();
            string sinkLabel = sinkVertexObject["label"]?.ToString();

            GraphViewJsonCommand.UpdateEdgeMetaProperty(outEdgeObject, edgeOffset, false, sinkId, sinkLabel);
            GraphViewJsonCommand.UpdateEdgeMetaProperty(inEdgeObject, edgeOffset, true, srcId, srcLabel);

            InsertEdgeObjectInternal(connection, srcVertexObject, srcVertexField, outEdgeObject, false, out outEdgeDocID); // srcVertex uploaded
            InsertEdgeObjectInternal(connection, sinkVertexObject, sinkVertexField, inEdgeObject, true, out inEdgeDocID);  // sinkVertex uploaded
        }
        internal override RawRecord DataModify(RawRecord record)
        {
            JObject vertexObject = this._vertexDocument;

            string vertexId = GraphViewConnection.GenerateDocumentId();

            Debug.Assert(vertexObject["id"] == null);
            Debug.Assert(vertexObject["_partition"] == null);
            vertexObject["id"]         = vertexId;
            vertexObject["_partition"] = vertexId;

            this.Connection.CreateDocumentAsync(vertexObject).Wait();

            VertexField vertexField = Connection.VertexCache.GetVertexField(vertexId, vertexObject);

            RawRecord result = new RawRecord();

            foreach (string fieldName in _projectedFieldList)
            {
                FieldObject fieldValue = vertexField[fieldName];

                result.Append(fieldValue);
            }

            return(result);
        }
        private void UpdatePropertiesOfVertex(VertexField vertex)
        {
            JObject vertexDocument = this.Connection.RetrieveDocumentById(vertex.VertexId);

            foreach (WPropertyExpression property in this.updateProperties)
            {
                Debug.Assert(property.Value != null);

                VertexPropertyField vertexProperty;
                string name = property.Key.Value;

                // Construct single property
                JObject meta = new JObject();
                foreach (KeyValuePair <WValueExpression, WValueExpression> pair in property.MetaProperties)
                {
                    meta[pair.Key.Value] = pair.Value.ToJValue();
                }
                JObject singleProperty = new JObject {
                    ["_value"]  = property.Value.ToJValue(),
                    ["_propId"] = GraphViewConnection.GenerateDocumentId(),
                    ["_meta"]   = meta,
                };

                // Set / Append to multiProperty
                JArray multiProperty;
                if (vertexDocument[name] == null)
                {
                    multiProperty        = new JArray();
                    vertexDocument[name] = multiProperty;
                }
                else
                {
                    multiProperty = (JArray)vertexDocument[name];
                }

                if (property.Cardinality == GremlinKeyword.PropertyCardinality.single)
                {
                    multiProperty.Clear();
                }
                multiProperty.Add(singleProperty);

                // Update vertex field
                bool existed = vertex.VertexProperties.TryGetValue(name, out vertexProperty);
                if (!existed)
                {
                    vertexProperty = new VertexPropertyField(vertexDocument.Property(name), vertex);
                    vertex.VertexProperties.Add(name, vertexProperty);
                }
                else
                {
                    vertexProperty.Replace(vertexDocument.Property(name));
                }
            }

            // Upload to DocDB
            this.Connection.ReplaceOrDeleteDocumentAsync(vertex.VertexId, vertexDocument, (string)vertexDocument["_partition"]).Wait();
        }
Ejemplo n.º 4
0
        public JObject ConstructNodeJsonDocument(string vertexLabel, List <WPropertyExpression> vertexProperties, out List <string> projectedFieldList)
        {
            Debug.Assert(vertexLabel != null);
            JObject vertexObject = new JObject {
                ["label"] = vertexLabel
            };

            projectedFieldList = new List <string>(GraphViewReservedProperties.ReservedNodeProperties);

            foreach (WPropertyExpression vertexProperty in vertexProperties)
            {
                Debug.Assert(vertexProperty.Cardinality == GremlinKeyword.PropertyCardinality.list);

                if (!projectedFieldList.Contains(vertexProperty.Key.Value))
                {
                    projectedFieldList.Add(vertexProperty.Key.Value);
                }

                if (vertexProperty.Value.ToJValue() == null)
                {
                    continue;
                }

                JObject meta = new JObject();
                foreach (KeyValuePair <WValueExpression, WValueExpression> pair in vertexProperty.MetaProperties)
                {
                    WValueExpression metaName  = pair.Key;
                    WValueExpression metaValue = pair.Value;
                    meta[metaName.Value] = metaValue.ToJValue();
                }

                string name      = vertexProperty.Key.Value;
                JArray propArray = (JArray)vertexObject[name];
                if (propArray == null)
                {
                    propArray          = new JArray();
                    vertexObject[name] = propArray;
                }

                propArray.Add(new JObject {
                    ["_value"]  = vertexProperty.Value.ToJValue(),
                    ["_propId"] = GraphViewConnection.GenerateDocumentId(),
                    ["_meta"]   = meta,
                });
                //GraphViewJsonCommand.AppendVertexSinglePropertyToVertex(vertexObject);
            }

            vertexObject["_edge"]           = new JArray();
            vertexObject["_reverse_edge"]   = new JArray();
            vertexObject["_nextEdgeOffset"] = 0;

            return(vertexObject);
        }
Ejemplo n.º 5
0
        public JObject ConstructNodeJsonDocument(string vertexLabel, List <WPropertyExpression> vertexProperties)
        {
            Debug.Assert(vertexLabel != null);
            JObject vertexObject = new JObject {
                [KW_VERTEX_LABEL] = vertexLabel
            };

            //projectedFieldList = new List<string>(GraphViewReservedProperties.InitialPopulateNodeProperties);
            //projectedFieldList.Add(GremlinKeyword.Label);

            foreach (WPropertyExpression vertexProperty in vertexProperties)
            {
                Debug.Assert(vertexProperty.Cardinality == GremlinKeyword.PropertyCardinality.List);

                //if (!projectedFieldList.Contains(vertexProperty.Key.Value))
                //    projectedFieldList.Add(vertexProperty.Key.Value);

                //if (vertexProperty.Value.ToJValue() == null) {
                //    continue;
                //}

                JObject meta = new JObject();
                foreach (KeyValuePair <WValueExpression, WValueExpression> pair in vertexProperty.MetaProperties)
                {
                    WValueExpression metaName  = pair.Key;
                    WValueExpression metaValue = pair.Value;
                    meta[metaName.Value] = metaValue.ToJValue();
                }

                string name      = vertexProperty.Key.Value;
                JArray propArray = (JArray)vertexObject[name];
                if (propArray == null)
                {
                    propArray          = new JArray();
                    vertexObject[name] = propArray;
                }

                propArray.Add(new JObject {
                    [KW_PROPERTY_VALUE] = vertexProperty.Value.ToJValue(),
                    [KW_PROPERTY_ID]    = GraphViewConnection.GenerateDocumentId(),
                    [KW_PROPERTY_META]  = meta,
                });
                //GraphViewJsonCommand.AppendVertexSinglePropertyToVertex(vertexObject);
            }

            vertexObject[KW_VERTEX_EDGE]            = new JArray();
            vertexObject[KW_VERTEX_REV_EDGE]        = new JArray();
            vertexObject[KW_VERTEX_EDGE_SPILLED]    = false;
            vertexObject[KW_VERTEX_REVEDGE_SPILLED] = false;
            //vertexObject[KW_VERTEX_NEXTOFFSET] = 0;

            return(vertexObject);
        }
Ejemplo n.º 6
0
        public JObject ConstructNodeJsonDocument(out List <string> projectedFieldList)
        {
            JObject nodeJsonDocument = new JObject();

            projectedFieldList = new List <string>(GraphViewReservedProperties.ReservedNodeProperties);

            for (var i = 0; i < Parameters.Count; i += 2)
            {
                var key = (Parameters[i] as WValueExpression).Value;

                //GraphViewJsonCommand.UpdateProperty(nodeJsonDocument, Parameters[i] as WValueExpression,
                //    Parameters[i + 1] as WValueExpression);
                GraphViewJsonCommand.UpdateProperty(nodeJsonDocument, Parameters[i] as WValueExpression,
                                                    Parameters[i + 1] as WValueExpression);
                string name  = (Parameters[i] as WValueExpression).Value;
                JToken value = (Parameters[i + 1] as WValueExpression).ToJValue();
                if (value != null)
                {
                    if (VertexField.IsVertexMetaProperty(name))
                    {
                        nodeJsonDocument[name] = value;
                    }
                    else
                    {
                        nodeJsonDocument[name] = new JArray {
                            new JObject {
                                ["_value"]  = value,
                                ["_propId"] = GraphViewConnection.GenerateDocumentId(),
                                ["_meta"]   = new JObject(),
                            },
                        };
                    }
                }

                if (!projectedFieldList.Contains(key))
                {
                    projectedFieldList.Add(key);
                }
            }

            //nodeJsonDocument = GraphViewJsonCommand.insert_property(nodeJsonDocument, "[]", "_edge").ToString();
            //nodeJsonDocument = GraphViewJsonCommand.insert_property(nodeJsonDocument, "[]", "_reverse_edge").ToString();
            nodeJsonDocument["_edge"]           = new JArray();
            nodeJsonDocument["_reverse_edge"]   = new JArray();
            nodeJsonDocument["_nextEdgeOffset"] = 0;

            return(nodeJsonDocument);
        }
Ejemplo n.º 7
0
        /// <summary>
        /// Add an edge from one vertex (source) to another (sink)
        /// NOTE: Both the source and sink vertex are modified.
        /// NOTE: This function may upload the edge-document.
        /// NOTE: srcVertex and sinkVertex are updated and uploaded.
        /// </summary>
        /// <param name="connection"></param>
        /// <param name="srcId"></param>
        /// <param name="sinkId"></param>
        /// <param name="srcVertexField"></param>
        /// <param name="sinkVertexField"></param>
        /// <param name="edgeJsonObject"></param>
        /// <param name="srcVertexObject"></param>
        /// <param name="sinkVertexObject"></param>
        /// <param name="outEdgeObject"></param>
        /// <param name="outEdgeDocID"></param>
        /// <param name="inEdgeObject"></param>
        /// <param name="inEdgeDocID"></param>
        public static void InsertEdgeAndUpload(
            GraphViewConnection connection,
            string srcId, string sinkId,
            VertexField srcVertexField, VertexField sinkVertexField,
            JObject edgeJsonObject,
            JObject srcVertexObject, JObject sinkVertexObject,
            out JObject outEdgeObject, out string outEdgeDocID,
            out JObject inEdgeObject, out string inEdgeDocID)
        {
            //long edgeOffset = (long)srcVertexObject[KW_VERTEX_NEXTOFFSET];
            //srcVertexObject[KW_VERTEX_NEXTOFFSET] = edgeOffset + 1;

            outEdgeObject = (JObject)edgeJsonObject.DeepClone();
            inEdgeObject  = (JObject)edgeJsonObject.DeepClone();

            // Add "id" property to edgeObject
            string edgeId = GraphViewConnection.GenerateDocumentId();

            string srcLabel  = srcVertexObject[KW_VERTEX_LABEL]?.ToString();
            string sinkLabel = sinkVertexObject[KW_VERTEX_LABEL]?.ToString();

            GraphViewJsonCommand.UpdateEdgeMetaProperty(outEdgeObject, edgeId, false, sinkId, sinkLabel);
            GraphViewJsonCommand.UpdateEdgeMetaProperty(inEdgeObject, edgeId, true, srcId, srcLabel);

            InsertEdgeObjectInternal(connection, srcVertexObject, srcVertexField, outEdgeObject, false, out outEdgeDocID); // srcVertex uploaded

            if (connection.UseReverseEdges)
            {
                InsertEdgeObjectInternal(connection, sinkVertexObject, sinkVertexField, inEdgeObject, true,
                                         out inEdgeDocID); // sinkVertex uploaded
            }
            else
            {
                inEdgeDocID = EdgeDocumentHelper.VirtualReverseEdgeDocId;
            }
        }
Ejemplo n.º 8
0
        /// <summary>
        /// This function spills a small-degree vertex, stores its edges into seperate documents
        /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size
        /// NOTE: This function will upload the vertex document
        /// </summary>
        /// <param name="connection"></param>
        /// <param name="vertexObject"></param>
        /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param>
        /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param>
        private static void SpillVertexEdgesToDocument(GraphViewConnection connection, JObject vertexObject, out string existEdgeDocId, out string newEdgeDocId)
        {
            Debug.Assert(vertexObject["_partition"] != null);
            Debug.Assert((string)vertexObject["id"] == (string)vertexObject["_partition"]);

            // NOTE: The VertexCache is not updated here
            bool outEdgeSeperated = vertexObject["_edge"] is JObject;
            bool inEdgeSeperated  = vertexObject["_reverse_edge"] is JObject;

            if (inEdgeSeperated && outEdgeSeperated)
            {
                throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated");
            }

            JArray targetEdgeArray;
            bool   targetEdgeIsReverse;

            if (inEdgeSeperated)
            {
                targetEdgeArray     = (JArray)vertexObject["_edge"];
                targetEdgeIsReverse = false;
            }
            else if (outEdgeSeperated)
            {
                targetEdgeArray     = (JArray)vertexObject["_reverse_edge"];
                targetEdgeIsReverse = true;
            }
            else
            {
                JArray outEdgeArray = (JArray)vertexObject["_edge"];
                JArray inEdgeArray  = (JArray)vertexObject["_reverse_edge"];
                targetEdgeIsReverse = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length);
                targetEdgeArray     = targetEdgeIsReverse ? inEdgeArray : outEdgeArray;
            }

            // Create a new edge-document to store the currently creating edge
            JObject newEdgeDocObject = new JObject {
                ["id"]          = GraphViewConnection.GenerateDocumentId(),
                ["_partition"]  = vertexObject["_partition"],
                ["_is_reverse"] = targetEdgeIsReverse,
                ["_is_reverse"] = targetEdgeIsReverse,
                ["_vertex_id"]  = (string)vertexObject["id"],
                ["_edge"]       = new JArray(targetEdgeArray.Last),
            };

            newEdgeDocId = connection.CreateDocumentAsync(newEdgeDocObject).Result;
            targetEdgeArray.Last.Remove();  // Remove the currently create edge appended just now

            // Create another new edge-document to store the existing edges.
            JObject existEdgeDocObject = new JObject {
                ["id"]          = GraphViewConnection.GenerateDocumentId(),
                ["_partition"]  = vertexObject["_partition"],
                ["_is_reverse"] = targetEdgeIsReverse,
                ["_vertex_id"]  = (string)vertexObject["id"],
                ["_edge"]       = targetEdgeArray,
            };

            existEdgeDocId = connection.CreateDocumentAsync(existEdgeDocObject).Result;

            // Update vertexObject to store the newly create edge-document & upload the vertexObject
            vertexObject[targetEdgeIsReverse ? "_reverse_edge" : "_edge"] = new JObject {
                ["_edges"] = new JArray {
                    new JObject {
                        ["id"] = existEdgeDocId,
                    },
                    new JObject {
                        ["id"] = newEdgeDocId,
                    },
                },
            };
            bool dummyTooLarge;

            UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge);
        }
Ejemplo n.º 9
0
        /// <summary>
        /// Insert edgeObject to one a vertex.
        /// NOTE: vertex-document and edge-document(s) are uploaded.
        /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
        ///       edges in VertexCache are updated (from null to the newly created edge-document's id)
        /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
        ///       if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
        ///       edge's property, but not adding a new edge.
        /// </summary>
        /// <param name="connection"></param>
        /// <param name="vertexObject"></param>
        /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
        /// <param name="edgeObject"></param>
        /// <param name="isReverse"></param>
        /// <param name="newEdgeDocId"></param>
        private static void InsertEdgeObjectInternal(
            GraphViewConnection connection,
            JObject vertexObject,
            VertexField vertexField,
            JObject edgeObject,
            bool isReverse,
            out string newEdgeDocId)
        {
            bool   tooLarge;
            JToken edgeContainer = vertexObject[isReverse ? "_reverse_edge" : "_edge"]; // JArray or JObject

            if (edgeContainer is JObject)
            {
                // Now it is a large-degree vertex, and contains at least 1 edge-document
                JArray edgeDocumentsArray = (JArray)edgeContainer["_edges"];
                Debug.Assert(edgeDocumentsArray != null, "edgeDocuments != null");
                Debug.Assert(edgeDocumentsArray.Count > 0, "edgeDocuments.Count > 0");

                string  lastEdgeDocId = (string)edgeDocumentsArray.Last["id"];
                JObject edgeDocument  = connection.RetrieveDocumentById(lastEdgeDocId);
                Debug.Assert(((string)edgeDocument["id"]).Equals(lastEdgeDocId), "((string)edgeDocument['id']).Equals(lastEdgeDocId)");
                Debug.Assert((bool)edgeDocument["_is_reverse"] == isReverse, "(bool)edgeDocument['_is_reverse'] == isReverse");
                Debug.Assert((string)edgeDocument["_vertex_id"] == (string)vertexObject["id"], "(string)edgeDocument['_vertex_id'] == (string)vertexObject['id']");

                JArray edgesArray = (JArray)edgeDocument["_edge"];
                Debug.Assert(edgesArray != null, "edgesArray != null");
                Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");

                if (connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    edgesArray.Add(edgeObject);
                    tooLarge = false;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    if (edgesArray.Count >= connection.EdgeSpillThreshold)
                    {
                        // The threshold is reached!
                        tooLarge = true;
                    }
                    else
                    {
                        // The threshold is not reached
                        edgesArray.Add(edgeObject);
                        tooLarge = false;
                    }
                }

                // If the edge-document is not too large (reach the threshold), try to
                //   upload the edge into the document
                if (!tooLarge)
                {
                    UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge);
                }
                if (tooLarge)
                {
                    // The edge is too large to be filled into the last edge-document
                    // or the threashold is reached:
                    // Create a new edge-document to store the edge.
                    JObject edgeDocObject = new JObject {
                        ["id"]          = GraphViewConnection.GenerateDocumentId(),
                        ["_partition"]  = vertexObject["_partition"],
                        ["_is_reverse"] = isReverse,
                        ["_vertex_id"]  = (string)vertexObject["id"],
                        ["_edge"]       = new JArray(edgeObject)
                    };
                    lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result;

                    // Add the newly create edge-document to vertexObject & upload the vertexObject
                    edgeDocumentsArray.Add(new JObject {
                        ["id"] = lastEdgeDocId
                    });
                }
                newEdgeDocId = lastEdgeDocId;

                // Upload the vertex documention (at least, its _nextXxx is changed)
                bool dummyTooLarge;
                UploadOne(connection, (string)vertexObject["id"], vertexObject, out dummyTooLarge);
                Debug.Assert(!dummyTooLarge);
            }
            else if (edgeContainer is JArray)
            {
                ((JArray)edgeContainer).Add(edgeObject);
                if (connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    tooLarge = false;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    tooLarge = (((JArray)edgeContainer).Count >= connection.EdgeSpillThreshold);
                }

                if (!tooLarge)
                {
                    UploadOne(connection, (string)vertexObject["id"], vertexObject, out tooLarge);
                }
                if (tooLarge)
                {
                    string existEdgeDocId;
                    SpillVertexEdgesToDocument(connection, vertexObject, out existEdgeDocId, out newEdgeDocId);

                    // Update the in & out edges in vertex field
                    if (isReverse)
                    {
                        Debug.Assert(vertexField.RevAdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null));
                        foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges)
                        {
                            edge.EdgeDocID = existEdgeDocId;
                        }
                    }
                    else
                    {
                        Debug.Assert(vertexField.AdjacencyList.AllEdges.All(edge => edge.EdgeDocID == null));
                        foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges)
                        {
                            edge.EdgeDocID = existEdgeDocId;
                        }
                    }
                }
                else
                {
                    newEdgeDocId = null;
                }
            }
            else
            {
                throw new Exception($"BUG: edgeContainer should either be JObject or JArray, but now: {edgeContainer?.GetType()}");
            }
        }
Ejemplo n.º 10
0
        /// <summary>
        /// This function spills a small-degree vertex, stores its edges into seperate documents
        /// Either its incoming or outgoing edges are moved to a new document, decided by which is larger in size
        /// NOTE: This function will upload the vertex document
        /// </summary>
        /// <param name="command"></param>
        /// <param name="vertexObject"></param>
        /// <param name="spillReverse">
        /// Whether to spill the outgoing edges or incoming edges.
        /// If it's null, let this function decide.
        /// (This happens when no spilling threshold is set but the document size limit is reached)
        /// </param>
        /// <param name="existEdgeDocId">This is the first edge-document (to store the existing edges)</param>
        /// <param name="newEdgeDocId">This is the second edge-document (to store the currently creating edge)</param>
        private static void SpillVertexEdgesToDocument(GraphViewCommand command, JObject vertexObject, ref bool?spillReverse, out string existEdgeDocId, out string newEdgeDocId)
        {
            if (spillReverse == null)
            {
                // Let this function decide whether incoming/outgoing edges to spill
                Debug.Assert(!IsSpilledVertex(vertexObject, true) || !IsSpilledVertex(vertexObject, false));
            }
            else
            {
                Debug.Assert(!IsSpilledVertex(vertexObject, spillReverse.Value));
            }

            // NOTE: The VertexCache is not updated here
            bool outEdgeSeperated = IsSpilledVertex(vertexObject, false);
            bool inEdgeSeperated  = IsSpilledVertex(vertexObject, true);

            if (inEdgeSeperated && outEdgeSeperated)
            {
                throw new Exception("BUG: Should not get here! Either incoming or outgoing edegs should not have been seperated");
            }

            JArray targetEdgeArray;

            if (inEdgeSeperated)
            {
                targetEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE];
                spillReverse    = false;
            }
            else if (outEdgeSeperated)
            {
                targetEdgeArray = (JArray)vertexObject[KW_VERTEX_REV_EDGE];
                spillReverse    = true;
            }
            else
            {
                JArray outEdgeArray = (JArray)vertexObject[KW_VERTEX_EDGE];
                JArray inEdgeArray  = (JArray)vertexObject[KW_VERTEX_REV_EDGE];
                spillReverse    = (outEdgeArray.ToString().Length < inEdgeArray.ToString().Length);
                targetEdgeArray = spillReverse.Value ? inEdgeArray : outEdgeArray;
            }

            // Create a new edge-document to store the currently creating edge
            JObject newEdgeDocObject = new JObject {
                [KW_DOC_ID]               = GraphViewConnection.GenerateDocumentId(),
                [KW_EDGEDOC_ISREVERSE]    = spillReverse.Value,
                [KW_EDGEDOC_VERTEXID]     = (string)vertexObject[KW_DOC_ID],
                [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
                [KW_EDGEDOC_EDGE]         = new JArray(targetEdgeArray.Last),
                [KW_EDGEDOC_IDENTIFIER]   = (JValue)true,
            };

            if (command.Connection.PartitionPathTopLevel != null)
            {
                newEdgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
            }

            newEdgeDocId = command.Connection.CreateDocumentAsync(newEdgeDocObject, command).Result;
            targetEdgeArray.Last.Remove();  // Remove the currently create edge appended just now

            // Create another new edge-document to store the existing edges.
            JObject existEdgeDocObject = new JObject {
                [KW_DOC_ID]               = GraphViewConnection.GenerateDocumentId(),
                [KW_EDGEDOC_ISREVERSE]    = spillReverse.Value,
                [KW_EDGEDOC_VERTEXID]     = (string)vertexObject[KW_DOC_ID],
                [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
                [KW_EDGEDOC_EDGE]         = targetEdgeArray,
                [KW_EDGEDOC_IDENTIFIER]   = (JValue)true,
            };

            if (command.Connection.PartitionPathTopLevel != null)
            {
                existEdgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
            }
            existEdgeDocId = command.Connection.CreateDocumentAsync(existEdgeDocObject, command).Result;

            // Update vertexObject to store the newly create edge-document & upload the vertexObject
            vertexObject[spillReverse.Value ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE] = new JArray {
                // Store the last spilled edge document only.
                new JObject {
                    [KW_DOC_ID] = newEdgeDocId,
                },
            };

            // Update the vertex document to indicate whether it's spilled
            if (spillReverse.Value)
            {
                Debug.Assert((bool)vertexObject[KW_VERTEX_REVEDGE_SPILLED] == false);
                vertexObject[KW_VERTEX_REVEDGE_SPILLED] = true;
            }
            else
            {
                Debug.Assert((bool)vertexObject[KW_VERTEX_EDGE_SPILLED] == false);
                vertexObject[KW_VERTEX_EDGE_SPILLED] = true;
            }

            bool dummyTooLarge;

            UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out dummyTooLarge);
            Debug.Assert(!dummyTooLarge);
        }
Ejemplo n.º 11
0
        /// <summary>
        /// Insert edgeObject to one a vertex.
        /// NOTE: vertex-document and edge-document(s) are uploaded.
        /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
        ///       edges in VertexCache are updated (from null to the newly created edge-document's id)
        /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
        ///       if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
        ///       edge's property, but not adding a new edge.
        /// </summary>
        /// <param name="command"></param>
        /// <param name="vertexObject"></param>
        /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
        /// <param name="edgeObject"></param>
        /// <param name="isReverse"></param>
        /// <param name="newEdgeDocId"></param>
        internal static void InsertEdgeObjectInternal(
            GraphViewCommand command,
            JObject vertexObject,
            VertexField vertexField,
            JObject edgeObject,
            bool isReverse,
            out string newEdgeDocId)
        {
            bool   tooLarge;
            JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject
            bool   isSpilled     = IsSpilledVertex(vertexObject, isReverse);

            //
            // This graph is compatible only, thus add an edge-document directly
            //
            if (command.Connection.GraphType != GraphType.GraphAPIOnly || command.Connection.EdgeSpillThreshold == 1)
            {
                Debug.Assert(command.Connection.EdgeSpillThreshold == 1);

                // Create a new edge-document to store the edge.
                JObject edgeDocObject = new JObject {
                    [KW_DOC_ID]               = GraphViewConnection.GenerateDocumentId(),
                    [KW_EDGEDOC_ISREVERSE]    = isReverse,
                    [KW_EDGEDOC_VERTEXID]     = (string)vertexObject[KW_DOC_ID],
                    [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
                    [KW_EDGEDOC_EDGE]         = new JArray(edgeObject),
                    [KW_EDGEDOC_IDENTIFIER]   = (JValue)true,
                };
                if (command.Connection.PartitionPathTopLevel != null)
                {
                    // This may be KW_DOC_PARTITION, maybe not
                    edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
                }

                // Upload the edge-document
                bool dummyTooLarge;
                UploadOne(command, (string)edgeDocObject[KW_DOC_ID], edgeDocObject, true, out dummyTooLarge);
                Debug.Assert(!dummyTooLarge);


                newEdgeDocId = (string)edgeDocObject[KW_DOC_ID];
                return;
            }


            if (isSpilled)
            {
                // Now it is a large-degree vertex, and contains at least 1 edge-document
                JArray edgeDocumentsArray = edgeContainer;
                Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null");
                Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1");

                string  lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID];
                JObject edgeDocument  = command.Connection.RetrieveDocumentById(lastEdgeDocId, vertexField.Partition, command);
                Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)");
                Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse");
                Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']");

                JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE];
                Debug.Assert(edgesArray != null, "edgesArray != null");
                Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");

                if (command.Connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    edgesArray.Add(edgeObject);
                    tooLarge = false;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    if (edgesArray.Count >= command.Connection.EdgeSpillThreshold)
                    {
                        // The threshold is reached!
                        tooLarge = true;
                    }
                    else
                    {
                        // The threshold is not reached
                        edgesArray.Add(edgeObject);
                        tooLarge = false;
                    }
                }

                // If the edge-document is not too large (reach the threshold), try to
                //   upload the edge into the document
                if (!tooLarge)
                {
                    UploadOne(command, lastEdgeDocId, edgeDocument, false, out tooLarge);
                }
                if (tooLarge)
                {
                    // The edge is too large to be filled into the last edge-document
                    // or the threashold is reached:
                    // Create a new edge-document to store the edge.
                    JObject edgeDocObject = new JObject {
                        [KW_DOC_ID]               = GraphViewConnection.GenerateDocumentId(),
                        [KW_EDGEDOC_ISREVERSE]    = isReverse,
                        [KW_EDGEDOC_VERTEXID]     = (string)vertexObject[KW_DOC_ID],
                        [KW_EDGEDOC_VERTEX_LABEL] = (string)vertexObject[KW_VERTEX_LABEL],
                        [KW_EDGEDOC_EDGE]         = new JArray(edgeObject),
                        [KW_EDGEDOC_IDENTIFIER]   = (JValue)true,
                    };
                    if (command.Connection.PartitionPathTopLevel != null)
                    {
                        // This may be KW_DOC_PARTITION, maybe not
                        edgeDocObject[command.Connection.PartitionPathTopLevel] = vertexObject[command.Connection.PartitionPathTopLevel];
                    }
                    lastEdgeDocId = command.Connection.CreateDocumentAsync(edgeDocObject, command).Result;

                    // Replace the newly created edge-document to vertexObject
                    Debug.Assert(edgeDocumentsArray.Count == 1);
                    edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId;
                }
                newEdgeDocId = lastEdgeDocId;

                // Upload the vertex documention (at least, its _nextXxx is changed)
                bool dummyTooLarge;
                UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out dummyTooLarge);
                Debug.Assert(!dummyTooLarge);
            }
            else
            {
                // This vertex is not spilled
                bool?spillReverse;
                ((JArray)edgeContainer).Add(edgeObject);
                if (command.Connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    tooLarge     = false;
                    spillReverse = null;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(command.Connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    tooLarge     = (((JArray)edgeContainer).Count > command.Connection.EdgeSpillThreshold);
                    spillReverse = isReverse;
                }

                if (!tooLarge)
                {
                    UploadOne(command, (string)vertexObject[KW_DOC_ID], vertexObject, false, out tooLarge);
                }
                if (tooLarge)
                {
                    string existEdgeDocId;
                    // The vertex object is uploaded in SpillVertexEdgesToDocument
                    EdgeDocumentHelper.SpillVertexEdgesToDocument(command, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId);

                    // the edges are spilled into two ducuments.
                    // one stores old edges(docId = existEdgeDocId), the other one stores the new edge.
                    // Because the new edge is not in the vertexCache, hence we can set all edges' docId as existEdgeDocId
                    Debug.Assert(spillReverse != null);
                    Debug.Assert(vertexField != null);
                    if (spillReverse.Value)
                    {
                        vertexField.RevAdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId);
                    }
                    else
                    {
                        vertexField.AdjacencyList.ResetFetchedEdgesDocId(existEdgeDocId);
                    }
                }
                else
                {
                    newEdgeDocId = null;
                }
            }
        }
Ejemplo n.º 12
0
        /// <summary>
        /// Insert edgeObject to one a vertex.
        /// NOTE: vertex-document and edge-document(s) are uploaded.
        /// NOTE: If changing _edge/_reverse_edge field from JArray to JObject, the "EdgeDocId" of existing
        ///       edges in VertexCache are updated (from null to the newly created edge-document's id)
        /// NOTE: Adding the newly created edge into VertexCache is not operated by this function. Actually,
        ///       if called by <see cref="UpdateEdgeProperty"/>, VertexCache should be updated by setting an
        ///       edge's property, but not adding a new edge.
        /// </summary>
        /// <param name="connection"></param>
        /// <param name="vertexObject"></param>
        /// <param name="vertexField">Can be null if we already know edgeContainer is JObject</param>
        /// <param name="edgeObject"></param>
        /// <param name="isReverse"></param>
        /// <param name="newEdgeDocId"></param>
        private static void InsertEdgeObjectInternal(
            GraphViewConnection connection,
            JObject vertexObject,
            VertexField vertexField,
            JObject edgeObject,
            bool isReverse,
            out string newEdgeDocId)
        {
            bool   tooLarge;
            JArray edgeContainer = (JArray)vertexObject[isReverse ? KW_VERTEX_REV_EDGE : KW_VERTEX_EDGE]; // JArray or JObject
            bool   isSpilled     = IsSpilledVertex(vertexObject, isReverse);

            if (isSpilled)
            {
                // Now it is a large-degree vertex, and contains at least 1 edge-document
                JArray edgeDocumentsArray = edgeContainer;
                Debug.Assert(edgeDocumentsArray != null, "edgeDocumentsArray != null");
                Debug.Assert(edgeDocumentsArray.Count == 1, "edgeDocumentsArray.Count == 1");

                string  lastEdgeDocId = (string)edgeDocumentsArray.Last[KW_DOC_ID];
                JObject edgeDocument  = connection.RetrieveDocumentById(lastEdgeDocId);
                Debug.Assert(((string)edgeDocument[KW_DOC_ID]).Equals(lastEdgeDocId), $"((string)edgeDocument[{KW_DOC_ID}]).Equals(lastEdgeDocId)");
                Debug.Assert((bool)edgeDocument[KW_EDGEDOC_ISREVERSE] == isReverse, $"(bool)edgeDocument['{KW_EDGEDOC_ISREVERSE}'] == isReverse");
                Debug.Assert((string)edgeDocument[KW_EDGEDOC_VERTEXID] == (string)vertexObject[KW_DOC_ID], $"(string)edgeDocument['{KW_EDGEDOC_VERTEXID}'] == (string)vertexObject['{KW_DOC_ID}']");

                JArray edgesArray = (JArray)edgeDocument[KW_EDGEDOC_EDGE];
                Debug.Assert(edgesArray != null, "edgesArray != null");
                Debug.Assert(edgesArray.Count > 0, "edgesArray.Count > 0");

                if (connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    edgesArray.Add(edgeObject);
                    tooLarge = false;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    if (edgesArray.Count >= connection.EdgeSpillThreshold)
                    {
                        // The threshold is reached!
                        tooLarge = true;
                    }
                    else
                    {
                        // The threshold is not reached
                        edgesArray.Add(edgeObject);
                        tooLarge = false;
                    }
                }

                // If the edge-document is not too large (reach the threshold), try to
                //   upload the edge into the document
                if (!tooLarge)
                {
                    UploadOne(connection, lastEdgeDocId, edgeDocument, out tooLarge);
                }
                if (tooLarge)
                {
                    // The edge is too large to be filled into the last edge-document
                    // or the threashold is reached:
                    // Create a new edge-document to store the edge.
                    JObject edgeDocObject = new JObject {
                        [KW_DOC_ID]            = GraphViewConnection.GenerateDocumentId(),
                        [KW_DOC_PARTITION]     = vertexObject[KW_DOC_PARTITION],
                        [KW_EDGEDOC_ISREVERSE] = isReverse,
                        [KW_EDGEDOC_VERTEXID]  = (string)vertexObject[KW_DOC_ID],
                        [KW_EDGEDOC_EDGE]      = new JArray(edgeObject)
                    };
                    lastEdgeDocId = connection.CreateDocumentAsync(edgeDocObject).Result;

                    //// Add the newly create edge-document to vertexObject & upload the vertexObject
                    //edgeDocumentsArray.Add(new JObject {
                    //    [KW_DOC_ID] = lastEdgeDocId
                    //});

                    // Replace the newly created edge-document to vertexObject
                    Debug.Assert(edgeDocumentsArray.Count == 1);
                    edgeDocumentsArray[0][KW_DOC_ID] = lastEdgeDocId;
                }
                newEdgeDocId = lastEdgeDocId;

                // Upload the vertex documention (at least, its _nextXxx is changed)
                bool dummyTooLarge;
                UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out dummyTooLarge);
                Debug.Assert(!dummyTooLarge);
            }
            else
            {
                // This vertex is not spilled
                bool?spillReverse;
                ((JArray)edgeContainer).Add(edgeObject);
                if (connection.EdgeSpillThreshold == 0)
                {
                    // Don't spill an edge-document until it is too large
                    tooLarge     = false;
                    spillReverse = null;
                }
                else
                {
                    // Explicitly specified a threshold
                    Debug.Assert(connection.EdgeSpillThreshold > 0, "connection.EdgeSpillThreshold > 0");
                    tooLarge     = (((JArray)edgeContainer).Count > connection.EdgeSpillThreshold);
                    spillReverse = isReverse;
                }

                if (!tooLarge)
                {
                    UploadOne(connection, (string)vertexObject[KW_DOC_ID], vertexObject, out tooLarge);
                }
                if (tooLarge)
                {
                    string existEdgeDocId;
                    // The vertex object is uploaded in SpillVertexEdgesToDocument
                    SpillVertexEdgesToDocument(connection, vertexObject, ref spillReverse, out existEdgeDocId, out newEdgeDocId);

                    // Update the in & out edges in vertex field
                    Debug.Assert(spillReverse != null);
                    Debug.Assert(vertexField != null);
                    if (spillReverse.Value)
                    {
                        foreach (EdgeField edge in vertexField.RevAdjacencyList.AllEdges)
                        {
                            Debug.Assert(edge.EdgeDocID == null);
                            edge.EdgeDocID = existEdgeDocId;
                        }
                    }
                    else
                    {
                        foreach (EdgeField edge in vertexField.AdjacencyList.AllEdges)
                        {
                            Debug.Assert(edge.EdgeDocID == null);
                            edge.EdgeDocID = existEdgeDocId;
                        }
                    }
                }
                else
                {
                    newEdgeDocId = null;
                }
            }
        }