/// <summary> /// Only if single Language/Location is defined per Campaign insert it as ExtraField of Campaign, otherwise - nothing /// Add Campaing and Language/Location of EdgeObjectsManager cache for futher Objects Import /// </summary> /// <param name="predicate">predicate to select only Language/Location objects</param> /// <param name="field"></param> /// <param name="isNegativeField"></param> private void PrepareImportObjects(Predicate <EdgeObject> predicate, EdgeField field, EdgeField isNegativeField) { foreach (var item in _campaignRelationMap.Where(x => x.Value.Count(y => predicate(y.Object2)) == 1)) { // get campaign and single object (Language or Location) defined for Campaign var campaign = _campaignRelationMap[item.Key][0].Object1 as Campaign; var obj = _campaignRelationMap[item.Key].FirstOrDefault(x => predicate(x.Object2)); if (campaign == null || obj == null || obj.Object2 == null) { continue; } if (campaign.Fields == null) { campaign.Fields = new Dictionary <EdgeField, object>(); } campaign.Fields.Add(field, obj.Object2); campaign.Fields.Add(isNegativeField, _campaignRelationMap[item.Key][0].IsNegative); ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject(campaign)); ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject(obj.Object2)); if (obj.Object2 is Location) { ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject((obj.Object2 as Location).LocationType)); } } }
private static void FindFieldDependencies(EdgeField field, Dictionary <EdgeField, EdgeFieldDependencyInfo> dependencies, List <EdgeField> edgeFields) { foreach (var childField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null)) { if (!dependencies[childField.Field].DependentFields.ContainsKey(field)) { if (childField.Field.FieldEdgeType.IsAbstract) { // take care of edge types inheritance foreach (var inheritor in GetInheritorsByType(childField.Field.FieldEdgeType.TypeID, edgeFields)) { dependencies[inheritor].DependentFields.Add(field, new EdgeTypeField { Field = field, ColumnName = childField.ColumnName, IsIdentity = childField.IsIdentity }); FindFieldDependencies(inheritor, dependencies, edgeFields); } } dependencies[childField.Field].DependentFields.Add(field, new EdgeTypeField { Field = field, ColumnName = childField.ColumnName, IsIdentity = childField.IsIdentity }); FindFieldDependencies(childField.Field, dependencies, edgeFields); } } }
private static int SetFieldDependencyDepth(EdgeField field, List <EdgeField> edgeFields) { var maxDepth = 0; foreach (var childField in field.FieldEdgeType.Fields) { if (childField.Field.FieldEdgeType != null) { if (childField.Field.FieldEdgeType.IsAbstract) { foreach (var inheritor in GetInheritorsByType(childField.Field.FieldEdgeType.TypeID, edgeFields)) { var childDepth = SetFieldDependencyDepth(inheritor, edgeFields); maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1; } } else { var childDepth = SetFieldDependencyDepth(childField.Field, edgeFields); maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1; } } } return(maxDepth); }
public static AdjacencyListField GetReverseAdjacencyListOfVertex(GraphViewConnection connection, string vertexId) { AdjacencyListField result = new AdjacencyListField(); string query = $"SELECT {{\"edge\": edge, " + $"\"_srcV\": doc.id, " + $"\"_srcVLabel\": doc.label}} AS incomingEdgeMetadata\n" + $"FROM doc\n" + $"JOIN edge IN doc._edge\n" + $"WHERE edge._sinkV = '{vertexId}'\n"; foreach (JObject edgeDocument in connection.ExecuteQuery(query)) { JObject edgeMetadata = (JObject)edgeDocument["incomingEdgeMetadata"]; JObject edgeObject = (JObject)edgeMetadata["edge"]; string srcV = edgeMetadata["_srcV"].ToString(); string srcVLabel = edgeMetadata["_srcVLabel"]?.ToString(); EdgeField edgeField = EdgeField.ConstructForwardEdgeField(srcV, srcVLabel, null, edgeObject); edgeField.EdgeProperties.Add("_srcV", new EdgePropertyField("_srcV", srcV, JsonDataType.String, edgeField)); if (srcVLabel != null) { edgeField.EdgeProperties.Add("_srcVLabel", new EdgePropertyField("_srcVLabel", srcVLabel, JsonDataType.String, edgeField)); } result.AddEdgeField(srcV, (long)edgeObject["_offset"], edgeField); } return(result); }
internal override List <RawRecord> CrossApply(RawRecord record) { List <RawRecord> results = new List <RawRecord>(); FieldObject target = record[this._targetIndex]; if (target != null) { VertexField vertex = target as VertexField; if (vertex != null) { RawRecord r = new RawRecord(); r.Append(new StringField(vertex.VertexMetaProperties[KW_DOC_ID].ToValue)); results.Add(r); } VertexSinglePropertyField vertexSingleProperty = target as VertexSinglePropertyField; if (vertexSingleProperty != null) { RawRecord r = new RawRecord(); r.Append(new StringField(vertexSingleProperty.PropertyId)); results.Add(r); } EdgeField edge = target as EdgeField; if (edge != null) { RawRecord r = new RawRecord(); r.Append(new StringField(edge.EdgeId)); results.Add(r); } } return(results); }
public DeltaEdgeField(EdgeField outEdgeField, VertexField srcVertexField, EdgeField inEdgeField, VertexField sinkVertexField, bool useReverseEdges) { this.outEdgeField = outEdgeField; this.inEdgeField = inEdgeField; this.srcVertexField = srcVertexField; this.sinkVertexField = sinkVertexField; this.UseReverseEdges = useReverseEdges; this.deltaProperties = new Dictionary <string, Tuple <EdgeDeltaType, JValue> >(); }
/// <summary> /// Before searching for object GKs update all GK of its parent fields /// (objects it depends on) /// For example: before searching for AdGroup GKs, update all AdGroup Campaings /// </summary> /// <param name="field"></param> private void UpdateObjectDependencies(EdgeField field) { // nothitng to do if there are no GK to update if (field.FieldEdgeType.Fields.All(x => x.Field.FieldEdgeType == null)) { return; } var mainTableName = GetDeliveryTableName(field.FieldEdgeType.TableName); var setStr = String.Format("UPDATE {0} \nSET ", mainTableName); var fromStr = "FROM "; var whereStr = String.Format("WHERE {0}.TYPEID=@typeId AND ", mainTableName); var paramList = new List <SqlParameter> { new SqlParameter("@typeId", field.FieldEdgeType.TypeID) }; foreach (var parentField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null)) { var tempParentTableName = String.Format("##TempDelivery_{0}", parentField.Field.Name); fromStr = String.Format("{0} {1},", fromStr, tempParentTableName); setStr = String.Format("{0} {1}={2}.GK,", setStr, parentField.ColumnNameGK, tempParentTableName); whereStr = String.Format("{0}{1}.{2}={3}.TK AND ", whereStr, mainTableName, parentField.ColumnNameTK, tempParentTableName); } if (fromStr.Length > 0) { fromStr = fromStr.Remove(fromStr.Length - 1, 1); } if (setStr.Length > 0) { setStr = setStr.Remove(setStr.Length - 1, 1); } if (whereStr.Length > 4) { whereStr = whereStr.Remove(whereStr.Length - 5, 5); } // perform update using (var cmd = new SqlCommand { Connection = _deliverySqlConnection }) { cmd.CommandText = String.Format("{0}\n{1}\n{2}", setStr, fromStr, whereStr); cmd.Parameters.AddRange(paramList.ToArray()); cmd.ExecuteNonQuery(); } }
private void DropEdge(EdgeField edgeField) { RawRecord record = new RawRecord(); record.Append(new StringField(edgeField.OutV)); // srcIdIndex record.Append(new StringField(edgeField.Offset.ToString())); // edgeOffsetIndex DropEdgeOperator op = new DropEdgeOperator(this.dummyInputOp, this.connection, 0, 1); op.DataModify(record); // Now VertexCacheObject has been updated (in DataModify) }
/// <summary> /// Create temporary table which contains GK, TK mapping of updated object type in Delivery /// </summary> /// <param name="field"></param> private void CreateTempGkTkTable4Field(EdgeField field) { using (var cmd = new SqlCommand { Connection = _deliverySqlConnection }) { cmd.CommandText = String.Format(@"SELECT GK, TK INTO ##TempDelivery_{0} FROM {1} WHERE TYPEID=@typeId; CREATE NONCLUSTERED INDEX [IDX_{0}_TK] ON ##TempDelivery_{0} (TK);" , field.Name, GetDeliveryTableName(field.FieldEdgeType.TableName)); cmd.Parameters.AddWithValue("@typeId", field.FieldEdgeType.TypeID); cmd.ExecuteNonQuery(); } }
private static int SetFieldDependencyDepth(EdgeField field) { var maxDepth = 0; foreach (var childField in field.FieldEdgeType.Fields) { if (childField.Field.FieldEdgeType != null) { var childDepth = SetFieldDependencyDepth(childField.Field); maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1; } } return(maxDepth); }
internal override RawRecord DataModify(RawRecord record) { FieldObject dropTarget = record[this.dropTargetIndex]; VertexField vertexField = dropTarget as VertexField;; if (vertexField != null) { this.DropVertex(vertexField); return(null); } EdgeField edgeField = dropTarget as EdgeField; if (edgeField != null) { this.DropEdge(edgeField); return(null); } PropertyField property = dropTarget as PropertyField; if (property != null) { if (property is VertexPropertyField) { this.DropVertexProperty((VertexPropertyField)property); } else if (property is VertexSinglePropertyField) { this.DropVertexSingleProperty((VertexSinglePropertyField)property); } else if (property is EdgePropertyField) { this.DropEdgeProperty((EdgePropertyField)property); } else { this.DropVertexPropertyMetaProperty((ValuePropertyField)property); } return(null); } // // Should not reach here // throw new Exception("BUG: Should not get here"); return(null); }
private static void FindFieldDependencies(EdgeField field, Dictionary <EdgeField, EdgeFieldDependencyInfo> dependencies) { foreach (var childField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null)) { if (!dependencies[childField.Field].DependentFields.ContainsKey(field)) { dependencies[childField.Field].DependentFields.Add(field, new EdgeTypeField { Field = field, ColumnName = childField.ColumnName, IsIdentity = childField.IsIdentity }); FindFieldDependencies(childField.Field, dependencies); } } }
public void Accumulate(params FieldObject[] values) { EdgeField edge = values[0] as EdgeField; if (edge == null) { throw new QueryExecutionException("Only edge can be an subgraph() input."); } this.subgraphState.edgeIds.Add(edge.EdgeId); this.subgraphState.vertexIds.Add(edge.InV); this.subgraphState.vertexIds.Add(edge.OutV); this.subgraphState.graph = null; }
public static Dictionary <string, AdjacencyListField> GetReverseAdjacencyListsOfVertexCollection(GraphViewConnection connection, HashSet <string> vertexIdSet) { Dictionary <string, AdjacencyListField> revAdjacencyListCollection = new Dictionary <string, AdjacencyListField>(); StringBuilder vertexIdList = new StringBuilder(); foreach (string vertexId in vertexIdSet) { if (vertexIdList.Length > 0) { vertexIdList.Append(", "); } vertexIdList.AppendFormat("'{0}'", vertexId); revAdjacencyListCollection[vertexId] = new AdjacencyListField(); } string query = $"SELECT {{\"edge\": edge, " + $"\"vertexId\": edge._sinkV, " + $"\"_srcV\": doc.id, " + $"\"_srcVLabel\": doc.label}} AS incomingEdgeMetadata\n" + $"FROM doc\n" + $"JOIN edge IN doc._edge\n" + $"WHERE edge._sinkV IN ({vertexIdList.ToString()})\n"; foreach (JObject edgeDocument in connection.ExecuteQuery(query)) { JObject edgeMetadata = (JObject)edgeDocument["incomingEdgeMetadata"]; JObject edgeObject = (JObject)edgeMetadata["edge"]; string vertexId = edgeMetadata["vertexId"].ToString(); string srcV = edgeMetadata["_srcV"].ToString(); string srcVLabel = edgeMetadata["_srcVLabel"]?.ToString(); EdgeField edgeField = EdgeField.ConstructForwardEdgeField(srcV, srcVLabel, null, edgeObject); edgeField.EdgeProperties.Add("_srcV", new EdgePropertyField("_srcV", srcV, JsonDataType.String, edgeField)); if (srcVLabel != null) { edgeField.EdgeProperties.Add("_srcVLabel", new EdgePropertyField("_srcVLabel", srcVLabel, JsonDataType.String, edgeField)); } AdjacencyListField revAdjList = revAdjacencyListCollection[vertexId]; revAdjList.AddEdgeField(srcV, (long)edgeObject["_offset"], edgeField); } return(revAdjacencyListCollection); }
/// <summary> /// Load edge fields by account + set edge type for each field /// </summary> public static List <EdgeField> LoadEdgeFields(int accountId, Dictionary <string, EdgeType> edgeTypes, SqlConnection connection) { var edgeFields = new List <EdgeField>(); try { using (var cmd = SqlUtility.CreateCommand("MD_EdgeField_Get", CommandType.StoredProcedure)) { cmd.Parameters.AddWithValue("@accountID", accountId); cmd.Connection = connection; using (var reader = cmd.ExecuteReader()) { while (reader.Read()) { EdgeField field = null; Type type = Type.GetType(reader["FieldType"].ToString()); if (type == null) { field = new SystemField(); } else { field = Activator.CreateInstance(type) as EdgeField; } if (field != null) { field.FieldID = int.Parse(reader["FieldID"].ToString()); field.Name = reader["Name"].ToString(); field.DisplayName = reader["DisplayName"].ToString(); field.FieldEdgeType = edgeTypes.Values.FirstOrDefault(x => x.TypeID == int.Parse(reader["FieldTypeID"].ToString())); edgeFields.Add(field); } } } } } catch (Exception ex) { throw new Exception("Error while trying to get extra fields from DB", ex); } return(edgeFields); }
internal void AddOrUpdateEdgeDelta(EdgeField outEdgeField, VertexField srcVertexField, EdgeField inEdgeField, VertexField sinkVertexField, DeltaLog log, bool useReverseEdges) { string edgeId = outEdgeField.EdgeId; DeltaEdgeField delta; if (edgeDelta.ContainsKey(edgeId)) { delta = edgeDelta[edgeId]; } else { delta = new DeltaEdgeField(outEdgeField, srcVertexField, inEdgeField, sinkVertexField, useReverseEdges); edgeDelta[edgeId] = delta; } delta.AddDeltaLog(log); }
internal override RawRecord DataModify(RawRecord record) { FieldObject updateTarget = record[this.updateTargetIndex]; VertexField vertex = updateTarget as VertexField;; if (vertex != null) { this.UpdatePropertiesOfVertex(vertex); return(record); } EdgeField edge = updateTarget as EdgeField; if (edge != null) { this.UpdatePropertiesOfEdge(edge); return(record); } PropertyField property = updateTarget as PropertyField; if (property != null) { if (property is VertexSinglePropertyField) { this.UpdateMetaPropertiesOfSingleVertexProperty((VertexSinglePropertyField)property); } else { throw new GraphViewException($"BUG: updateTarget is {nameof(PropertyField)}: {property.GetType()}"); } return(record); } // // Should not reach here // throw new Exception("BUG: Should not get here!"); }
public override FieldObject Evaluate(RawRecord record) { FieldObject checkObject = record[_checkFieldIndex]; VertexField vertexField = checkObject as VertexField; EdgeField edgeField = checkObject as EdgeField; if (vertexField != null) { if (vertexField.AllProperties.Count(pf => pf.PropertyName == _propertyName) > 0) { return(new StringField("true", JsonDataType.Boolean)); } else { return(new StringField("false", JsonDataType.Boolean)); } } else if (edgeField != null) { if (edgeField.EdgeProperties.ContainsKey(_propertyName)) { return(new StringField("true", JsonDataType.Boolean)); } else { return(new StringField("false", JsonDataType.Boolean)); } } else { throw new GraphViewException( "HasProperty() function can only be applied to a VertexField or EdgeField but now the object is " + checkObject.GetType()); } }
private void UpdatePropertiesOfEdge(EdgeField edge) { List <Tuple <WValueExpression, WValueExpression, int> > propertyList = new List <Tuple <WValueExpression, WValueExpression, int> >(); foreach (WPropertyExpression property in this.updateProperties) { if (property.Cardinality == GremlinKeyword.PropertyCardinality.list || property.MetaProperties.Count > 0) { throw new Exception("Can't create meta property or duplicated property on edges"); } propertyList.Add(new Tuple <WValueExpression, WValueExpression, int>(property.Key, property.Value, 0)); } RawRecord record = new RawRecord(); record.Append(new StringField(edge.OutV)); record.Append(new StringField(edge.Offset.ToString())); UpdateEdgePropertiesOperator op = new UpdateEdgePropertiesOperator(this.InputOperator, this.Connection, 0, 1, propertyList); op.DataModify(record); }
internal override RawRecord DataModify(RawRecord record) { VertexField srcVertexField = _srcFunction.Evaluate(record) as VertexField; VertexField sinkVertexField = _sinkFunction.Evaluate(record) as VertexField; if (srcVertexField == null || sinkVertexField == null) { return(null); } string srcId = srcVertexField["id"].ToValue; string sinkId = sinkVertexField["id"].ToValue; //string srcJsonDocument = srcVertexField.JsonDocument; //string sinkJsonDocument = sinkVertexField.JsonDocument; JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcId); JObject sinkVertexObject; if (srcId.Equals(sinkId)) { // MUST not use JObject.DeepClone() here! sinkVertexObject = srcVertexObject; } else { sinkVertexObject = this.Connection.RetrieveDocumentById(sinkId); } //VertexField srcVertexField = (srcFieldObject as VertexField) // ?? Connection.VertexCache.GetVertexField(srcId, srcVertexObject); //VertexField sinkVertexField = (sinkFieldObject as VertexField) // ?? Connection.VertexCache.GetVertexField(sinkId, sinkVertexObject); // // Interact with DocDB and add the edge // - For a small-degree vertex (now filled into one document), insert the edge in-place // - If the upload succeeds, done! // - If the upload fails with size-limit-exceeded(SLE), put either incoming or outgoing edges into a seperate document // - For a large-degree vertex (already spilled) // - Update either incoming or outgoing edges in the seperate edge-document // - If the upload fails with SLE, create a new document to store the edge, and update the vertex document // JObject outEdgeObject, inEdgeObject; string outEdgeDocID, inEdgeDocID; EdgeDocumentHelper.InsertEdgeAndUpload(this.Connection, srcId, sinkId, srcVertexField, sinkVertexField, this._edgeJsonObject, srcVertexObject, sinkVertexObject, out outEdgeObject, out outEdgeDocID, out inEdgeObject, out inEdgeDocID); // // Update vertex's adjacency list and reverse adjacency list (in vertex field) // EdgeField outEdgeField = EdgeField.ConstructForwardEdgeField(srcId, srcVertexField["label"]?.ToValue, outEdgeDocID, outEdgeObject); EdgeField inEdgeField = EdgeField.ConstructBackwardEdgeField(sinkId, sinkVertexField["label"]?.ToValue, inEdgeDocID, inEdgeObject); srcVertexField.AdjacencyList.AddEdgeField(srcId, outEdgeField.Offset, outEdgeField); sinkVertexField.RevAdjacencyList.AddEdgeField(srcId, inEdgeField.Offset, inEdgeField); // Construct the newly added edge's RawRecord RawRecord result = new RawRecord(); // source, sink, other, offset, * result.Append(new StringField(srcId)); result.Append(new StringField(sinkId)); result.Append(new StringField(_otherVTag == 0 ? srcId : sinkId)); result.Append(new StringField(outEdgeObject["_offset"].ToString())); result.Append(outEdgeField); for (int i = GraphViewReservedProperties.ReservedEdgeProperties.Count; i < _edgeProperties.Count; i++) { FieldObject fieldValue = outEdgeField[_edgeProperties[i]]; result.Append(fieldValue); } return(result); }
internal override List <RawRecord> CrossApply(RawRecord record) { List <RawRecord> results = new List <RawRecord>(); FieldObject inputTarget = record[this.inputTargetIndex]; if (inputTarget is VertexField) { VertexField vertexField = (VertexField)inputTarget; foreach (VertexPropertyField property in vertexField.VertexProperties.Values) { string propertyName = property.PropertyName; Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName)); Debug.Assert(!propertyName.Equals(KW_VERTEX_EDGE)); Debug.Assert(!propertyName.Equals(KW_VERTEX_REV_EDGE)); switch (propertyName) { case "_rid": case "_self": case "_etag": case "_attachments": case "_ts": continue; default: foreach (VertexSinglePropertyField singleVp in property.Multiples.Values) { RawRecord r = new RawRecord(); r.Append(new VertexSinglePropertyField(singleVp)); foreach (string metaPropertyName in this.populateMetaProperties) { r.Append(singleVp[metaPropertyName]); } results.Add(r); } break; } } } else if (inputTarget is EdgeField) { EdgeField edgeField = (EdgeField)inputTarget; foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties) { string propertyName = propertyPair.Key; EdgePropertyField edgePropertyField = propertyPair.Value; switch (propertyName) { // Reserved properties for meta-data case KW_EDGE_LABEL: case KW_EDGE_ID: //case KW_EDGE_OFFSET: case KW_EDGE_SRCV: case KW_EDGE_SINKV: case KW_EDGE_SRCV_LABEL: case KW_EDGE_SINKV_LABEL: continue; default: RawRecord r = new RawRecord(); r.Append(new EdgePropertyField(edgePropertyField)); results.Add(r); break; } } if (this.populateMetaProperties.Count > 0 && results.Count > 0) { throw new GraphViewException("An edge property cannot contain meta properties."); } } else if (inputTarget is VertexSinglePropertyField) { VertexSinglePropertyField singleVp = (VertexSinglePropertyField)inputTarget; foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties) { RawRecord r = new RawRecord(); ValuePropertyField metaPropertyField = kvp.Value; r.Append(new ValuePropertyField(metaPropertyField)); results.Add(r); } if (this.populateMetaProperties.Count > 0 && results.Count > 0) { throw new GraphViewException("An edge property cannot contain meta properties."); } } else { throw new GraphViewException("The input of properties() cannot be a meta or edge property."); } return(results); }
internal override List <RawRecord> CrossApply(RawRecord record) { MapField valueMap = new MapField(); FieldObject inputTarget = record[this.inputTargetIndex]; if (inputTarget is VertexField) { VertexField vertexField = (VertexField)inputTarget; if (this.propertyNameList.Any()) { foreach (string propertyName in this.propertyNameList) { FieldObject property = vertexField[propertyName]; if (property == null) { continue; } List <FieldObject> values = new List <FieldObject>(); VertexPropertyField vp = property as VertexPropertyField; if (vp != null) { foreach (VertexSinglePropertyField vsp in vp.Multiples.Values) { values.Add(vsp); } } valueMap.Add(new StringField(propertyName), new CollectionField(values)); } } else { foreach (VertexPropertyField property in vertexField.VertexProperties.Values) { string propertyName = property.PropertyName; Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName)); Debug.Assert(!propertyName.Equals(KW_VERTEX_EDGE)); Debug.Assert(!propertyName.Equals(KW_VERTEX_REV_EDGE)); switch (propertyName) { case "_rid": case "_self": case "_etag": case "_attachments": case "_ts": continue; default: List <FieldObject> values = new List <FieldObject>(); foreach (VertexSinglePropertyField singleVp in property.Multiples.Values) { values.Add(singleVp); } valueMap.Add(new StringField(propertyName), new CollectionField(values)); break; } } } } else if (inputTarget is EdgeField) { EdgeField edgeField = (EdgeField)inputTarget; if (this.propertyNameList.Any()) { foreach (string propertyName in this.propertyNameList) { FieldObject property = edgeField[propertyName]; if (property == null) { continue; } EdgePropertyField edgePf = property as EdgePropertyField; if (edgePf != null) { valueMap.Add(new StringField(propertyName), edgePf); } } } else { foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties) { string propertyName = propertyPair.Key; EdgePropertyField edgePropertyField = propertyPair.Value; switch (propertyName) { // Reserved properties for meta-data case KW_EDGE_ID: //case KW_EDGE_OFFSET: case KW_EDGE_SRCV: case KW_EDGE_SINKV: case KW_EDGE_SRCV_LABEL: case KW_EDGE_SINKV_LABEL: continue; default: valueMap.Add(new StringField(propertyName), edgePropertyField); break; } } } } else if (inputTarget is VertexSinglePropertyField) { VertexSinglePropertyField singleVp = inputTarget as VertexSinglePropertyField; if (this.propertyNameList.Any()) { foreach (string propertyName in this.propertyNameList) { FieldObject property = singleVp[propertyName]; if (property == null) { continue; } ValuePropertyField metaPf = property as ValuePropertyField; if (metaPf != null) { valueMap.Add(new StringField(propertyName), metaPf); } } } else { foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties) { valueMap.Add(new StringField(kvp.Key), kvp.Value); } } } else { throw new GraphViewException("The input of valueMap() cannot be a meta or edge property."); } RawRecord result = new RawRecord(); result.Append(valueMap); return(new List <RawRecord> { result }); }
public IEnumerator <Tuple <VertexField, RawRecord> > GetVerticesAndEdgesViaVertices(JsonQuery vertexQuery, GraphViewCommand command) { IEnumerable <JObject> items = this.ExecuteQueryScript(vertexQuery); List <string> nodeProperties = new List <string>(vertexQuery.NodeProperties); List <string> edgeProperties = new List <string>(vertexQuery.EdgeProperties); string nodeAlias = nodeProperties[0]; // Skip i = 0, which is the (node.* as nodeAlias) field nodeProperties.RemoveAt(0); // // TODO: Refactor // string edgeAlias = null; bool isReverseAdj = false; bool isStartVertexTheOriginVertex = false; bool crossApplyEdgeOnServer = edgeProperties.Any(); if (crossApplyEdgeOnServer) { edgeAlias = edgeProperties[0]; isReverseAdj = bool.Parse(edgeProperties[1]); isStartVertexTheOriginVertex = bool.Parse(edgeProperties[2]); edgeProperties.RemoveAt(0); edgeProperties.RemoveAt(0); edgeProperties.RemoveAt(0); } // // Batch strategy: // - For "small" vertexes, they have been cross applied on the server side // - For "large" vertexes, just return the VertexField, the adjacency list decoder will // construct spilled adjacency lists in batch mode and cross apply edges after that // Func <VertexField, string, Tuple <VertexField, RawRecord> > makeCrossAppliedRecord = (vertexField, edgeId) => { Debug.Assert(vertexField != null); RawRecord nodeRecord = new RawRecord(); // // Fill node property field // foreach (string propertyName in nodeProperties) { FieldObject propertyValue = vertexField[propertyName]; nodeRecord.Append(propertyValue); } RawRecord edgeRecord = new RawRecord(edgeProperties.Count); EdgeField edgeField = ((AdjacencyListField)vertexField[isReverseAdj ? DocumentDBKeywords.KW_VERTEX_REV_EDGE : DocumentDBKeywords.KW_VERTEX_EDGE]) .GetEdgeField(edgeId, true); string startVertexId = vertexField.VertexId; AdjacencyListDecoder.FillMetaField(edgeRecord, edgeField, startVertexId, vertexField.Partition, isStartVertexTheOriginVertex, isReverseAdj); AdjacencyListDecoder.FillPropertyField(edgeRecord, edgeField, edgeProperties); nodeRecord.Append(edgeRecord); return(new Tuple <VertexField, RawRecord>(vertexField, nodeRecord)); }; Func <VertexField, Tuple <VertexField, RawRecord> > makeRawRecord = (vertexField) => { Debug.Assert(vertexField != null); RawRecord rawRecord = new RawRecord(); // // Fill node property field // foreach (string propertyName in nodeProperties) { FieldObject propertyValue = vertexField[propertyName]; rawRecord.Append(propertyValue); } return(new Tuple <VertexField, RawRecord>(vertexField, rawRecord)); }; HashSet <string> uniqueVertexIds = new HashSet <string>(); HashSet <string> uniqueEdgeIds = new HashSet <string>(); foreach (JObject dynamicItem in items) { JObject tmpVertexObject = (JObject)((JObject)dynamicItem)[nodeAlias]; string vertexId = (string)tmpVertexObject[DocumentDBKeywords.KW_DOC_ID]; if (crossApplyEdgeOnServer) { // Note: since vertex properties can be multi-valued, // a DocumentDB query needs a join clause in the FROM clause // to retrieve vertex property values, which may result in // the same vertex being returned multiple times. // We use the hash set uniqueVertexIds to ensure one vertex is // produced only once. if (EdgeDocumentHelper.IsBuildingTheAdjacencyListLazily( tmpVertexObject, isReverseAdj, this.Connection.UseReverseEdges) && uniqueVertexIds.Add(vertexId)) { VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject); yield return(makeRawRecord(vertexField)); } else // When the DocumentDB query crosses apply edges { JObject edgeObjct = (JObject)((JObject)dynamicItem)[edgeAlias]; string edgeId = (string)edgeObjct[DocumentDBKeywords.KW_EDGE_ID]; if (uniqueEdgeIds.Add(edgeId)) { VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject); yield return(makeCrossAppliedRecord(vertexField, edgeId)); } } } else { if (!uniqueVertexIds.Add(vertexId)) { continue; } VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject); yield return(makeRawRecord(vertexField)); } } }
public IEnumerator <RawRecord> GetVerticesAndEdgesViaEdges(JsonQuery edgeQuery, GraphViewCommand command) { IEnumerable <JObject> items = this.ExecuteQueryScript(edgeQuery); List <string> nodeProperties = new List <string>(edgeQuery.NodeProperties); List <string> edgeProperties = new List <string>(edgeQuery.EdgeProperties); string nodeAlias = nodeProperties[0]; nodeProperties.RemoveAt(0); string edgeAlias = edgeProperties[0]; edgeProperties.RemoveAt(0); HashSet <string> spilledVertexIdSet = new HashSet <string>(); HashSet <string> spilledVertexPartitionSet = new HashSet <string>(); // // <vertex id, edge id> // Dictionary <string, List <string> > vertexIdAndEdgeIdsDict = new Dictionary <string, List <string> >(); // // <vertex id, <edgeDocumentId, edgeObject>> // Dictionary <string, List <Tuple <string, JObject> > > vertexIdAndEdgeObjectsDict = new Dictionary <string, List <Tuple <string, JObject> > >(); foreach (JObject dynamicItem in items) { JObject tmpObject = (JObject)dynamicItem[nodeAlias]; JObject edgeObject = (JObject)dynamicItem[edgeAlias]; // // This is a spilled edge document // if (tmpObject[DocumentDBKeywords.KW_EDGEDOC_VERTEXID] != null) { string vertexId = tmpObject[DocumentDBKeywords.KW_EDGEDOC_VERTEXID].ToString(); spilledVertexIdSet.Add(vertexId); string partition = this.Connection.GetDocumentPartition(tmpObject); if (partition != null) { spilledVertexPartitionSet.Add(partition); } List <Tuple <string, JObject> > edgeObjects; if (!vertexIdAndEdgeObjectsDict.TryGetValue(vertexId, out edgeObjects)) { edgeObjects = new List <Tuple <string, JObject> >(); vertexIdAndEdgeObjectsDict.Add(vertexId, edgeObjects); } edgeObjects.Add(new Tuple <string, JObject>((string)tmpObject[DocumentDBKeywords.KW_DOC_ID], edgeObject)); List <string> edgeIds; if (!vertexIdAndEdgeIdsDict.TryGetValue(vertexId, out edgeIds)) { edgeIds = new List <string>(); vertexIdAndEdgeIdsDict.Add(vertexId, edgeIds); } edgeIds.Add((string)edgeObject[DocumentDBKeywords.KW_DOC_ID]); } else { string vertexId = (string)tmpObject[DocumentDBKeywords.KW_DOC_ID]; command.VertexCache.AddOrUpdateVertexField(vertexId, tmpObject); List <string> edgeIds; if (!vertexIdAndEdgeIdsDict.TryGetValue(vertexId, out edgeIds)) { edgeIds = new List <string>(); vertexIdAndEdgeIdsDict.Add(vertexId, edgeIds); } edgeIds.Add((string)edgeObject[DocumentDBKeywords.KW_DOC_ID]); } } if (spilledVertexIdSet.Any()) { const string NODE_ALISE = "Node"; JsonQuery query = new JsonQuery { NodeAlias = NODE_ALISE }; query.AddSelectElement("*"); query.RawWhereClause = new WInPredicate(new WColumnReferenceExpression(NODE_ALISE, "id"), spilledVertexIdSet.ToList()); if (spilledVertexPartitionSet.Any()) { query.WhereConjunction( new WInPredicate( new WValueExpression($"{NODE_ALISE}{this.Connection.GetPartitionPathIndexer()}"), spilledVertexPartitionSet.ToList()), BooleanBinaryExpressionType.And); } // TODO: remove this code when you understand it. // string idInClause = string.Join(", ", spilledVertexIdSet.Select(id => $"'{id}'")); // string partitionInClause = string.Join(", ", spilledVertexPartitionSet.Select(partition => $"'{partition}'")); // string queryScript = $"SELECT * FROM Node WHERE Node.id IN ({idInClause})" + // (string.IsNullOrEmpty(partitionInClause) // ? "" // : $" AND Node{this.Connection.GetPartitionPathIndexer()} IN ({partitionInClause})"); // IEnumerable<dynamic> spilledVertices = this.Connection.ExecuteDocDbQuery(queryScript); IEnumerable <JObject> spilledVertices = this.ExecuteQueryScript(query); foreach (JObject vertex in spilledVertices) { JObject vertexObject = vertex; string vertexId = (string)vertexObject[DocumentDBKeywords.KW_DOC_ID]; VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, vertexObject); vertexField.ConstructPartialLazyAdjacencyList(vertexIdAndEdgeObjectsDict[vertexId], false); } } foreach (KeyValuePair <string, List <string> > pair in vertexIdAndEdgeIdsDict) { string vertexId = pair.Key; List <string> edgeIds = pair.Value; VertexField vertexField = command.VertexCache.GetVertexField(vertexId); foreach (string edgeId in edgeIds) { RawRecord nodeRecord = new RawRecord(); // // Fill node property field // foreach (string propertyName in nodeProperties) { FieldObject propertyValue = vertexField[propertyName]; nodeRecord.Append(propertyValue); } RawRecord edgeRecord = new RawRecord(edgeProperties.Count); EdgeField edgeField = vertexField.AdjacencyList.GetEdgeField(edgeId, false); Debug.Assert(edgeField != null, "edgeField != null"); string startVertexId = vertexField.VertexId; AdjacencyListDecoder.FillMetaField(edgeRecord, edgeField, startVertexId, vertexField.Partition, true, false); AdjacencyListDecoder.FillPropertyField(edgeRecord, edgeField, edgeProperties); nodeRecord.Append(edgeRecord); yield return(nodeRecord); } } }
internal override List <RawRecord> CrossApply(RawRecord record) { var results = new List <RawRecord>(); // Extract all properties if allPropertyIndex >= 0 if (allPropertyIndex >= 0 && record[allPropertyIndex] != null) { VertexField vertexField = record[allPropertyIndex] as VertexField; if (vertexField != null) { foreach (PropertyField property in vertexField.AllProperties) { string propertyName = property.PropertyName; switch (propertyName) { // Reversed properties for meta-data case "_edge": case "_partition": case "_reverse_edge": case "_nextEdgeOffset": case "_rid": case "_self": case "_etag": case "_attachments": case "_ts": continue; default: RawRecord r = new RawRecord(); if (property is VertexSinglePropertyField || property is ValuePropertyField) { r.Append(property); } else if (property is VertexPropertyField) { foreach (VertexSinglePropertyField p in ((VertexPropertyField)property).Multiples.Values) { r.Append(p); } } else { Debug.Assert(false, $"[PropertiesOperator.CrossApply] property type error: {property.GetType()}"); } results.Add(r); break; } } } else { EdgeField edgeField = record[allPropertyIndex] as EdgeField; if (edgeField == null) { throw new GraphViewException( string.Format("The FieldObject record[{0}] should be a VertexField or EdgeField but now it is {1}.", allPropertyIndex, record[allPropertyIndex].ToString())); } foreach (var propertyPair in edgeField.EdgeProperties) { string propertyName = propertyPair.Key; EdgePropertyField propertyField = propertyPair.Value; switch (propertyName) { // Reversed properties for meta-data case "_offset": case "_srcV": case "_sinkV": case "_srcVLabel": case "_sinkVLabel": continue; default: RawRecord r = new RawRecord(); r.Append(propertyField); results.Add(r); break; } } } } else { // TODO: Now translation code needn't to generate the key name for the operator foreach (var pair in propertyList) { //string propertyName = pair.Item1; int propertyValueIndex = pair.Item2; var propertyValue = record[propertyValueIndex]; if (propertyValue == null) { continue; } var result = new RawRecord(); result.Append(propertyValue); results.Add(result); } } return(results); }
internal override List <RawRecord> CrossApply(RawRecord record) { var results = new List <RawRecord>(); // Extract all values if allValuesIndex >= 0 if (allValuesIndex >= 0 && record[allValuesIndex] != null) { VertexField vertexField = record[allValuesIndex] as VertexField; if (vertexField != null) { foreach (PropertyField property in vertexField.AllProperties) { string propertyName = property.PropertyName; switch (propertyName) { // Reversed properties for meta-data case "_edge": case "_reverse_edge": case "_partition": case "_nextEdgeOffset": case "_rid": case "_self": case "_etag": case "_attachments": case "_ts": continue; default: RawRecord r = new RawRecord(); r.Append(new StringField(property.PropertyValue, property.JsonDataType)); results.Add(r); break; } } } else { EdgeField edgeField = record[allValuesIndex] as EdgeField; if (edgeField == null) { throw new GraphViewException( string.Format("The FieldObject record[{0}] should be a VertexField or EdgeField but now it is {1}.", allValuesIndex, record[allValuesIndex].ToString())); } foreach (var propertyPair in edgeField.EdgeProperties) { string propertyName = propertyPair.Key; EdgePropertyField propertyField = propertyPair.Value; switch (propertyName) { // Reversed properties for meta-data case "_offset": case "_srcV": case "_srcVLabel": case "_sinkV": case "_sinkVLabel": continue; default: RawRecord r = new RawRecord(); r.Append(new StringField(propertyField.ToValue, propertyField.JsonDataType)); results.Add(r); break; } } } } else { foreach (var propIdx in ValuesIdxList) { PropertyField propertyValue = record[propIdx] as PropertyField; if (propertyValue == null) { continue; } var result = new RawRecord(); result.Append(new StringField(propertyValue.ToValue, propertyValue.JsonDataType)); results.Add(result); } } return(results); }
internal override List <RawRecord> CrossApply(RawRecord record) { List <RawRecord> results = new List <RawRecord>(); FieldObject inputTarget = record[this.inputTargetIndex]; if (inputTarget is VertexField) { VertexField vertexField = (VertexField)inputTarget; foreach (VertexPropertyField property in vertexField.VertexProperties.Values) { string propertyName = property.PropertyName; Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName)); Debug.Assert(propertyName == "_edge"); Debug.Assert(propertyName == "_reverse_edge"); switch (propertyName) { case "_rid": case "_self": case "_etag": case "_attachments": case "_ts": continue; default: foreach (VertexSinglePropertyField singleVp in property.Multiples.Values) { RawRecord r = new RawRecord(); r.Append(new StringField(singleVp.PropertyValue, singleVp.JsonDataType)); results.Add(r); } break; } } } else if (inputTarget is EdgeField) { EdgeField edgeField = (EdgeField)inputTarget; foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties) { string propertyName = propertyPair.Key; EdgePropertyField edgePropertyField = propertyPair.Value; switch (propertyName) { // Reserved properties for meta-data case "_edgeId": case "_offset": case "_srcV": case "_sinkV": case "_srcVLabel": case "_sinkVLabel": continue; default: RawRecord r = new RawRecord(); r.Append(new StringField(edgePropertyField.PropertyValue, edgePropertyField.JsonDataType)); results.Add(r); break; } } } else if (inputTarget is VertexSinglePropertyField) { VertexSinglePropertyField singleVp = inputTarget as VertexSinglePropertyField; foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties) { RawRecord r = new RawRecord(); ValuePropertyField metaPropertyField = kvp.Value; r.Append(new StringField(metaPropertyField.PropertyValue, metaPropertyField.JsonDataType)); results.Add(r); } } else { throw new GraphViewException("The input of values() cannot be a meta or edge property."); } return(results); }
internal override RawRecord DataModify(RawRecord record) { long edgeOffset = long.Parse(record[this._edgeOffsetIndex].ToValue); string srcVertexId = record[this._srcVertexIdIndex].ToValue; JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcVertexId); string outEdgeDocId; JObject outEdgeObject; EdgeDocumentHelper.FindEdgeBySourceAndOffset( this.Connection, srcVertexObject, srcVertexId, edgeOffset, false, out outEdgeObject, out outEdgeDocId); if (outEdgeObject == null) { // TODO: Is there something wrong? Debug.WriteLine($"[UpdateEdgePropertiesOperator] The edge does not exist: vertexId = {srcVertexId}, edgeOffset = {edgeOffset}"); return(null); } string sinkVertexId = (string)outEdgeObject["_sinkV"]; JObject sinkVertexObject; string inEdgeDocId; JObject inEdgeObject; if (sinkVertexId.Equals(srcVertexId)) { sinkVertexObject = srcVertexObject; // NOTE: Must not use DeepClone() here! } else { sinkVertexObject = this.Connection.RetrieveDocumentById(sinkVertexId); } EdgeDocumentHelper.FindEdgeBySourceAndOffset( this.Connection, sinkVertexObject, srcVertexId, edgeOffset, true, out inEdgeObject, out inEdgeDocId); VertexField srcVertexField = this.Connection.VertexCache.GetVertexField(srcVertexId); VertexField sinkVertexField = this.Connection.VertexCache.GetVertexField(sinkVertexId); EdgeField outEdgeField = srcVertexField.AdjacencyList.GetEdgeField(srcVertexId, edgeOffset); EdgeField inEdgeField = sinkVertexField.RevAdjacencyList.GetEdgeField(srcVertexId, edgeOffset); // Drop all non-reserved properties if (this.PropertiesToBeUpdated.Count == 1 && !this.PropertiesToBeUpdated[0].Item1.SingleQuoted && this.PropertiesToBeUpdated[0].Item1.Value.Equals("*", StringComparison.OrdinalIgnoreCase) && !this.PropertiesToBeUpdated[0].Item2.SingleQuoted && this.PropertiesToBeUpdated[0].Item2.Value.Equals("null", StringComparison.OrdinalIgnoreCase)) { List <string> toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(outEdgeObject); foreach (var propertyName in toBeDroppedProperties) { outEdgeField.EdgeProperties.Remove(propertyName); } toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(inEdgeObject); foreach (var propertyName in toBeDroppedProperties) { inEdgeField.EdgeProperties.Remove(propertyName); } } else { foreach (Tuple <WValueExpression, WValueExpression, int> tuple in this.PropertiesToBeUpdated) { WValueExpression keyExpression = tuple.Item1; WValueExpression valueExpression = tuple.Item2; if (this.Mode == UpdatePropertyMode.Set) { // Modify edgeObject (update the edge property) JProperty updatedProperty = GraphViewJsonCommand.UpdateProperty(outEdgeObject, keyExpression, valueExpression); // Update VertexCache if (updatedProperty == null) { outEdgeField.EdgeProperties.Remove(keyExpression.Value); } else { outEdgeField.UpdateEdgeProperty(updatedProperty, outEdgeField); } // Modify edgeObject (update the edge property) updatedProperty = GraphViewJsonCommand.UpdateProperty(inEdgeObject, keyExpression, valueExpression); // Update VertexCache if (updatedProperty == null) { inEdgeField.EdgeProperties.Remove(keyExpression.Value); } else { inEdgeField.UpdateEdgeProperty(updatedProperty, inEdgeField); } } else { throw new NotImplementedException(); } } } // Interact with DocDB to update the property EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, srcVertexObject, outEdgeDocId, false, outEdgeObject); EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, sinkVertexObject, inEdgeDocId, true, inEdgeObject); // // Drop edge property // if (this.PropertiesToBeUpdated.Any(t => t.Item2 == null)) { return(null); } return(record); }