/// <summary>
/// Only if single Language/Location is defined per Campaign insert it as ExtraField of Campaign, otherwise - nothing
/// Add Campaing and Language/Location of EdgeObjectsManager cache for futher Objects Import
/// </summary>
/// <param name="predicate">predicate to select only Language/Location objects</param>
/// <param name="field"></param>
/// <param name="isNegativeField"></param>
private void PrepareImportObjects(Predicate <EdgeObject> predicate, EdgeField field, EdgeField isNegativeField)
{
foreach (var item in _campaignRelationMap.Where(x => x.Value.Count(y => predicate(y.Object2)) == 1))
{
// get campaign and single object (Language or Location) defined for Campaign
var campaign = _campaignRelationMap[item.Key][0].Object1 as Campaign;
var obj = _campaignRelationMap[item.Key].FirstOrDefault(x => predicate(x.Object2));
if (campaign == null || obj == null || obj.Object2 == null)
{
continue;
}
if (campaign.Fields == null)
{
campaign.Fields = new Dictionary <EdgeField, object>();
}
campaign.Fields.Add(field, obj.Object2);
campaign.Fields.Add(isNegativeField, _campaignRelationMap[item.Key][0].IsNegative);
ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject(campaign));
ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject(obj.Object2));
if (obj.Object2 is Location)
{
ImportManager.EdgeObjectsManager.AddToCache(NormalizeEdgeObject((obj.Object2 as Location).LocationType));
}
}
}
private static void FindFieldDependencies(EdgeField field, Dictionary <EdgeField, EdgeFieldDependencyInfo> dependencies, List <EdgeField> edgeFields)
{
foreach (var childField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null))
{
if (!dependencies[childField.Field].DependentFields.ContainsKey(field))
{
if (childField.Field.FieldEdgeType.IsAbstract)
{
// take care of edge types inheritance
foreach (var inheritor in GetInheritorsByType(childField.Field.FieldEdgeType.TypeID, edgeFields))
{
dependencies[inheritor].DependentFields.Add(field, new EdgeTypeField
{
Field = field,
ColumnName = childField.ColumnName,
IsIdentity = childField.IsIdentity
});
FindFieldDependencies(inheritor, dependencies, edgeFields);
}
}
dependencies[childField.Field].DependentFields.Add(field, new EdgeTypeField
{
Field = field,
ColumnName = childField.ColumnName,
IsIdentity = childField.IsIdentity
});
FindFieldDependencies(childField.Field, dependencies, edgeFields);
}
}
}
private static int SetFieldDependencyDepth(EdgeField field, List <EdgeField> edgeFields)
{
var maxDepth = 0;
foreach (var childField in field.FieldEdgeType.Fields)
{
if (childField.Field.FieldEdgeType != null)
{
if (childField.Field.FieldEdgeType.IsAbstract)
{
foreach (var inheritor in GetInheritorsByType(childField.Field.FieldEdgeType.TypeID, edgeFields))
{
var childDepth = SetFieldDependencyDepth(inheritor, edgeFields);
maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1;
}
}
else
{
var childDepth = SetFieldDependencyDepth(childField.Field, edgeFields);
maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1;
}
}
}
return(maxDepth);
}
public static AdjacencyListField GetReverseAdjacencyListOfVertex(GraphViewConnection connection, string vertexId)
{
AdjacencyListField result = new AdjacencyListField();
string query = $"SELECT {{\"edge\": edge, " +
$"\"_srcV\": doc.id, " +
$"\"_srcVLabel\": doc.label}} AS incomingEdgeMetadata\n" +
$"FROM doc\n" +
$"JOIN edge IN doc._edge\n" +
$"WHERE edge._sinkV = '{vertexId}'\n";
foreach (JObject edgeDocument in connection.ExecuteQuery(query))
{
JObject edgeMetadata = (JObject)edgeDocument["incomingEdgeMetadata"];
JObject edgeObject = (JObject)edgeMetadata["edge"];
string srcV = edgeMetadata["_srcV"].ToString();
string srcVLabel = edgeMetadata["_srcVLabel"]?.ToString();
EdgeField edgeField = EdgeField.ConstructForwardEdgeField(srcV, srcVLabel, null, edgeObject);
edgeField.EdgeProperties.Add("_srcV", new EdgePropertyField("_srcV", srcV, JsonDataType.String, edgeField));
if (srcVLabel != null)
{
edgeField.EdgeProperties.Add("_srcVLabel",
new EdgePropertyField("_srcVLabel", srcVLabel, JsonDataType.String, edgeField));
}
result.AddEdgeField(srcV, (long)edgeObject["_offset"], edgeField);
}
return(result);
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
List <RawRecord> results = new List <RawRecord>();
FieldObject target = record[this._targetIndex];
if (target != null)
{
VertexField vertex = target as VertexField;
if (vertex != null)
{
RawRecord r = new RawRecord();
r.Append(new StringField(vertex.VertexMetaProperties[KW_DOC_ID].ToValue));
results.Add(r);
}
VertexSinglePropertyField vertexSingleProperty = target as VertexSinglePropertyField;
if (vertexSingleProperty != null)
{
RawRecord r = new RawRecord();
r.Append(new StringField(vertexSingleProperty.PropertyId));
results.Add(r);
}
EdgeField edge = target as EdgeField;
if (edge != null)
{
RawRecord r = new RawRecord();
r.Append(new StringField(edge.EdgeId));
results.Add(r);
}
}
return(results);
}
public DeltaEdgeField(EdgeField outEdgeField, VertexField srcVertexField, EdgeField inEdgeField, VertexField sinkVertexField, bool useReverseEdges)
{
this.outEdgeField = outEdgeField;
this.inEdgeField = inEdgeField;
this.srcVertexField = srcVertexField;
this.sinkVertexField = sinkVertexField;
this.UseReverseEdges = useReverseEdges;
this.deltaProperties = new Dictionary <string, Tuple <EdgeDeltaType, JValue> >();
}
/// <summary>
/// Before searching for object GKs update all GK of its parent fields
/// (objects it depends on)
/// For example: before searching for AdGroup GKs, update all AdGroup Campaings
/// </summary>
/// <param name="field"></param>
private void UpdateObjectDependencies(EdgeField field)
{
// nothitng to do if there are no GK to update
if (field.FieldEdgeType.Fields.All(x => x.Field.FieldEdgeType == null))
{
return;
}
var mainTableName = GetDeliveryTableName(field.FieldEdgeType.TableName);
var setStr = String.Format("UPDATE {0} \nSET ", mainTableName);
var fromStr = "FROM ";
var whereStr = String.Format("WHERE {0}.TYPEID=@typeId AND ", mainTableName);
var paramList = new List <SqlParameter> {
new SqlParameter("@typeId", field.FieldEdgeType.TypeID)
};
foreach (var parentField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null))
{
var tempParentTableName = String.Format("##TempDelivery_{0}", parentField.Field.Name);
fromStr = String.Format("{0} {1},", fromStr, tempParentTableName);
setStr = String.Format("{0} {1}={2}.GK,",
setStr,
parentField.ColumnNameGK,
tempParentTableName);
whereStr = String.Format("{0}{1}.{2}={3}.TK AND ",
whereStr,
mainTableName,
parentField.ColumnNameTK,
tempParentTableName);
}
if (fromStr.Length > 0)
{
fromStr = fromStr.Remove(fromStr.Length - 1, 1);
}
if (setStr.Length > 0)
{
setStr = setStr.Remove(setStr.Length - 1, 1);
}
if (whereStr.Length > 4)
{
whereStr = whereStr.Remove(whereStr.Length - 5, 5);
}
// perform update
using (var cmd = new SqlCommand {
Connection = _deliverySqlConnection
})
{
cmd.CommandText = String.Format("{0}\n{1}\n{2}", setStr, fromStr, whereStr);
cmd.Parameters.AddRange(paramList.ToArray());
cmd.ExecuteNonQuery();
}
}
private void DropEdge(EdgeField edgeField)
{
RawRecord record = new RawRecord();
record.Append(new StringField(edgeField.OutV)); // srcIdIndex
record.Append(new StringField(edgeField.Offset.ToString())); // edgeOffsetIndex
DropEdgeOperator op = new DropEdgeOperator(this.dummyInputOp, this.connection, 0, 1);
op.DataModify(record);
// Now VertexCacheObject has been updated (in DataModify)
}
/// <summary>
/// Create temporary table which contains GK, TK mapping of updated object type in Delivery
/// </summary>
/// <param name="field"></param>
private void CreateTempGkTkTable4Field(EdgeField field)
{
using (var cmd = new SqlCommand {
Connection = _deliverySqlConnection
})
{
cmd.CommandText = String.Format(@"SELECT GK, TK INTO ##TempDelivery_{0} FROM {1} WHERE TYPEID=@typeId;
CREATE NONCLUSTERED INDEX [IDX_{0}_TK] ON ##TempDelivery_{0} (TK);" ,
field.Name, GetDeliveryTableName(field.FieldEdgeType.TableName));
cmd.Parameters.AddWithValue("@typeId", field.FieldEdgeType.TypeID);
cmd.ExecuteNonQuery();
}
}
private static int SetFieldDependencyDepth(EdgeField field)
{
var maxDepth = 0;
foreach (var childField in field.FieldEdgeType.Fields)
{
if (childField.Field.FieldEdgeType != null)
{
var childDepth = SetFieldDependencyDepth(childField.Field);
maxDepth = maxDepth > childDepth + 1 ? maxDepth : childDepth + 1;
}
}
return(maxDepth);
}
internal override RawRecord DataModify(RawRecord record)
{
FieldObject dropTarget = record[this.dropTargetIndex];
VertexField vertexField = dropTarget as VertexField;;
if (vertexField != null)
{
this.DropVertex(vertexField);
return(null);
}
EdgeField edgeField = dropTarget as EdgeField;
if (edgeField != null)
{
this.DropEdge(edgeField);
return(null);
}
PropertyField property = dropTarget as PropertyField;
if (property != null)
{
if (property is VertexPropertyField)
{
this.DropVertexProperty((VertexPropertyField)property);
}
else if (property is VertexSinglePropertyField)
{
this.DropVertexSingleProperty((VertexSinglePropertyField)property);
}
else if (property is EdgePropertyField)
{
this.DropEdgeProperty((EdgePropertyField)property);
}
else
{
this.DropVertexPropertyMetaProperty((ValuePropertyField)property);
}
return(null);
}
//
// Should not reach here
//
throw new Exception("BUG: Should not get here");
return(null);
}
private static void FindFieldDependencies(EdgeField field, Dictionary <EdgeField, EdgeFieldDependencyInfo> dependencies)
{
foreach (var childField in field.FieldEdgeType.Fields.Where(x => x.Field.FieldEdgeType != null))
{
if (!dependencies[childField.Field].DependentFields.ContainsKey(field))
{
dependencies[childField.Field].DependentFields.Add(field, new EdgeTypeField
{
Field = field,
ColumnName = childField.ColumnName,
IsIdentity = childField.IsIdentity
});
FindFieldDependencies(childField.Field, dependencies);
}
}
}
public void Accumulate(params FieldObject[] values)
{
EdgeField edge = values[0] as EdgeField;
if (edge == null)
{
throw new QueryExecutionException("Only edge can be an subgraph() input.");
}
this.subgraphState.edgeIds.Add(edge.EdgeId);
this.subgraphState.vertexIds.Add(edge.InV);
this.subgraphState.vertexIds.Add(edge.OutV);
this.subgraphState.graph = null;
}
public static Dictionary <string, AdjacencyListField> GetReverseAdjacencyListsOfVertexCollection(GraphViewConnection connection, HashSet <string> vertexIdSet)
{
Dictionary <string, AdjacencyListField> revAdjacencyListCollection = new Dictionary <string, AdjacencyListField>();
StringBuilder vertexIdList = new StringBuilder();
foreach (string vertexId in vertexIdSet)
{
if (vertexIdList.Length > 0)
{
vertexIdList.Append(", ");
}
vertexIdList.AppendFormat("'{0}'", vertexId);
revAdjacencyListCollection[vertexId] = new AdjacencyListField();
}
string query = $"SELECT {{\"edge\": edge, " +
$"\"vertexId\": edge._sinkV, " +
$"\"_srcV\": doc.id, " +
$"\"_srcVLabel\": doc.label}} AS incomingEdgeMetadata\n" +
$"FROM doc\n" +
$"JOIN edge IN doc._edge\n" +
$"WHERE edge._sinkV IN ({vertexIdList.ToString()})\n";
foreach (JObject edgeDocument in connection.ExecuteQuery(query))
{
JObject edgeMetadata = (JObject)edgeDocument["incomingEdgeMetadata"];
JObject edgeObject = (JObject)edgeMetadata["edge"];
string vertexId = edgeMetadata["vertexId"].ToString();
string srcV = edgeMetadata["_srcV"].ToString();
string srcVLabel = edgeMetadata["_srcVLabel"]?.ToString();
EdgeField edgeField = EdgeField.ConstructForwardEdgeField(srcV, srcVLabel, null, edgeObject);
edgeField.EdgeProperties.Add("_srcV", new EdgePropertyField("_srcV", srcV, JsonDataType.String, edgeField));
if (srcVLabel != null)
{
edgeField.EdgeProperties.Add("_srcVLabel",
new EdgePropertyField("_srcVLabel", srcVLabel, JsonDataType.String, edgeField));
}
AdjacencyListField revAdjList = revAdjacencyListCollection[vertexId];
revAdjList.AddEdgeField(srcV, (long)edgeObject["_offset"], edgeField);
}
return(revAdjacencyListCollection);
}
/// <summary>
/// Load edge fields by account + set edge type for each field
/// </summary>
public static List <EdgeField> LoadEdgeFields(int accountId, Dictionary <string, EdgeType> edgeTypes, SqlConnection connection)
{
var edgeFields = new List <EdgeField>();
try
{
using (var cmd = SqlUtility.CreateCommand("MD_EdgeField_Get", CommandType.StoredProcedure))
{
cmd.Parameters.AddWithValue("@accountID", accountId);
cmd.Connection = connection;
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
EdgeField field = null;
Type type = Type.GetType(reader["FieldType"].ToString());
if (type == null)
{
field = new SystemField();
}
else
{
field = Activator.CreateInstance(type) as EdgeField;
}
if (field != null)
{
field.FieldID = int.Parse(reader["FieldID"].ToString());
field.Name = reader["Name"].ToString();
field.DisplayName = reader["DisplayName"].ToString();
field.FieldEdgeType = edgeTypes.Values.FirstOrDefault(x => x.TypeID == int.Parse(reader["FieldTypeID"].ToString()));
edgeFields.Add(field);
}
}
}
}
}
catch (Exception ex)
{
throw new Exception("Error while trying to get extra fields from DB", ex);
}
return(edgeFields);
}
internal void AddOrUpdateEdgeDelta(EdgeField outEdgeField, VertexField srcVertexField,
EdgeField inEdgeField, VertexField sinkVertexField, DeltaLog log, bool useReverseEdges)
{
string edgeId = outEdgeField.EdgeId;
DeltaEdgeField delta;
if (edgeDelta.ContainsKey(edgeId))
{
delta = edgeDelta[edgeId];
}
else
{
delta = new DeltaEdgeField(outEdgeField, srcVertexField, inEdgeField, sinkVertexField, useReverseEdges);
edgeDelta[edgeId] = delta;
}
delta.AddDeltaLog(log);
}
internal override RawRecord DataModify(RawRecord record)
{
FieldObject updateTarget = record[this.updateTargetIndex];
VertexField vertex = updateTarget as VertexField;;
if (vertex != null)
{
this.UpdatePropertiesOfVertex(vertex);
return(record);
}
EdgeField edge = updateTarget as EdgeField;
if (edge != null)
{
this.UpdatePropertiesOfEdge(edge);
return(record);
}
PropertyField property = updateTarget as PropertyField;
if (property != null)
{
if (property is VertexSinglePropertyField)
{
this.UpdateMetaPropertiesOfSingleVertexProperty((VertexSinglePropertyField)property);
}
else
{
throw new GraphViewException($"BUG: updateTarget is {nameof(PropertyField)}: {property.GetType()}");
}
return(record);
}
//
// Should not reach here
//
throw new Exception("BUG: Should not get here!");
}
public override FieldObject Evaluate(RawRecord record)
{
FieldObject checkObject = record[_checkFieldIndex];
VertexField vertexField = checkObject as VertexField;
EdgeField edgeField = checkObject as EdgeField;
if (vertexField != null)
{
if (vertexField.AllProperties.Count(pf => pf.PropertyName == _propertyName) > 0)
{
return(new StringField("true", JsonDataType.Boolean));
}
else
{
return(new StringField("false", JsonDataType.Boolean));
}
}
else if (edgeField != null)
{
if (edgeField.EdgeProperties.ContainsKey(_propertyName))
{
return(new StringField("true", JsonDataType.Boolean));
}
else
{
return(new StringField("false", JsonDataType.Boolean));
}
}
else
{
throw new GraphViewException(
"HasProperty() function can only be applied to a VertexField or EdgeField but now the object is " +
checkObject.GetType());
}
}
private void UpdatePropertiesOfEdge(EdgeField edge)
{
List <Tuple <WValueExpression, WValueExpression, int> > propertyList =
new List <Tuple <WValueExpression, WValueExpression, int> >();
foreach (WPropertyExpression property in this.updateProperties)
{
if (property.Cardinality == GremlinKeyword.PropertyCardinality.list ||
property.MetaProperties.Count > 0)
{
throw new Exception("Can't create meta property or duplicated property on edges");
}
propertyList.Add(new Tuple <WValueExpression, WValueExpression, int>(property.Key, property.Value, 0));
}
RawRecord record = new RawRecord();
record.Append(new StringField(edge.OutV));
record.Append(new StringField(edge.Offset.ToString()));
UpdateEdgePropertiesOperator op = new UpdateEdgePropertiesOperator(this.InputOperator, this.Connection, 0, 1, propertyList);
op.DataModify(record);
}
internal override RawRecord DataModify(RawRecord record)
{
VertexField srcVertexField = _srcFunction.Evaluate(record) as VertexField;
VertexField sinkVertexField = _sinkFunction.Evaluate(record) as VertexField;
if (srcVertexField == null || sinkVertexField == null)
{
return(null);
}
string srcId = srcVertexField["id"].ToValue;
string sinkId = sinkVertexField["id"].ToValue;
//string srcJsonDocument = srcVertexField.JsonDocument;
//string sinkJsonDocument = sinkVertexField.JsonDocument;
JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcId);
JObject sinkVertexObject;
if (srcId.Equals(sinkId))
{
// MUST not use JObject.DeepClone() here!
sinkVertexObject = srcVertexObject;
}
else
{
sinkVertexObject = this.Connection.RetrieveDocumentById(sinkId);
}
//VertexField srcVertexField = (srcFieldObject as VertexField)
// ?? Connection.VertexCache.GetVertexField(srcId, srcVertexObject);
//VertexField sinkVertexField = (sinkFieldObject as VertexField)
// ?? Connection.VertexCache.GetVertexField(sinkId, sinkVertexObject);
//
// Interact with DocDB and add the edge
// - For a small-degree vertex (now filled into one document), insert the edge in-place
// - If the upload succeeds, done!
// - If the upload fails with size-limit-exceeded(SLE), put either incoming or outgoing edges into a seperate document
// - For a large-degree vertex (already spilled)
// - Update either incoming or outgoing edges in the seperate edge-document
// - If the upload fails with SLE, create a new document to store the edge, and update the vertex document
//
JObject outEdgeObject, inEdgeObject;
string outEdgeDocID, inEdgeDocID;
EdgeDocumentHelper.InsertEdgeAndUpload(this.Connection,
srcId, sinkId,
srcVertexField, sinkVertexField, this._edgeJsonObject,
srcVertexObject, sinkVertexObject,
out outEdgeObject, out outEdgeDocID,
out inEdgeObject, out inEdgeDocID);
//
// Update vertex's adjacency list and reverse adjacency list (in vertex field)
//
EdgeField outEdgeField = EdgeField.ConstructForwardEdgeField(srcId, srcVertexField["label"]?.ToValue, outEdgeDocID, outEdgeObject);
EdgeField inEdgeField = EdgeField.ConstructBackwardEdgeField(sinkId, sinkVertexField["label"]?.ToValue, inEdgeDocID, inEdgeObject);
srcVertexField.AdjacencyList.AddEdgeField(srcId, outEdgeField.Offset, outEdgeField);
sinkVertexField.RevAdjacencyList.AddEdgeField(srcId, inEdgeField.Offset, inEdgeField);
// Construct the newly added edge's RawRecord
RawRecord result = new RawRecord();
// source, sink, other, offset, *
result.Append(new StringField(srcId));
result.Append(new StringField(sinkId));
result.Append(new StringField(_otherVTag == 0 ? srcId : sinkId));
result.Append(new StringField(outEdgeObject["_offset"].ToString()));
result.Append(outEdgeField);
for (int i = GraphViewReservedProperties.ReservedEdgeProperties.Count; i < _edgeProperties.Count; i++)
{
FieldObject fieldValue = outEdgeField[_edgeProperties[i]];
result.Append(fieldValue);
}
return(result);
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
List <RawRecord> results = new List <RawRecord>();
FieldObject inputTarget = record[this.inputTargetIndex];
if (inputTarget is VertexField)
{
VertexField vertexField = (VertexField)inputTarget;
foreach (VertexPropertyField property in vertexField.VertexProperties.Values)
{
string propertyName = property.PropertyName;
Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName));
Debug.Assert(!propertyName.Equals(KW_VERTEX_EDGE));
Debug.Assert(!propertyName.Equals(KW_VERTEX_REV_EDGE));
switch (propertyName)
{
case "_rid":
case "_self":
case "_etag":
case "_attachments":
case "_ts":
continue;
default:
foreach (VertexSinglePropertyField singleVp in property.Multiples.Values)
{
RawRecord r = new RawRecord();
r.Append(new VertexSinglePropertyField(singleVp));
foreach (string metaPropertyName in this.populateMetaProperties)
{
r.Append(singleVp[metaPropertyName]);
}
results.Add(r);
}
break;
}
}
}
else if (inputTarget is EdgeField)
{
EdgeField edgeField = (EdgeField)inputTarget;
foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties)
{
string propertyName = propertyPair.Key;
EdgePropertyField edgePropertyField = propertyPair.Value;
switch (propertyName)
{
// Reserved properties for meta-data
case KW_EDGE_LABEL:
case KW_EDGE_ID:
//case KW_EDGE_OFFSET:
case KW_EDGE_SRCV:
case KW_EDGE_SINKV:
case KW_EDGE_SRCV_LABEL:
case KW_EDGE_SINKV_LABEL:
continue;
default:
RawRecord r = new RawRecord();
r.Append(new EdgePropertyField(edgePropertyField));
results.Add(r);
break;
}
}
if (this.populateMetaProperties.Count > 0 && results.Count > 0)
{
throw new GraphViewException("An edge property cannot contain meta properties.");
}
}
else if (inputTarget is VertexSinglePropertyField)
{
VertexSinglePropertyField singleVp = (VertexSinglePropertyField)inputTarget;
foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties)
{
RawRecord r = new RawRecord();
ValuePropertyField metaPropertyField = kvp.Value;
r.Append(new ValuePropertyField(metaPropertyField));
results.Add(r);
}
if (this.populateMetaProperties.Count > 0 && results.Count > 0)
{
throw new GraphViewException("An edge property cannot contain meta properties.");
}
}
else
{
throw new GraphViewException("The input of properties() cannot be a meta or edge property.");
}
return(results);
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
MapField valueMap = new MapField();
FieldObject inputTarget = record[this.inputTargetIndex];
if (inputTarget is VertexField)
{
VertexField vertexField = (VertexField)inputTarget;
if (this.propertyNameList.Any())
{
foreach (string propertyName in this.propertyNameList)
{
FieldObject property = vertexField[propertyName];
if (property == null)
{
continue;
}
List <FieldObject> values = new List <FieldObject>();
VertexPropertyField vp = property as VertexPropertyField;
if (vp != null)
{
foreach (VertexSinglePropertyField vsp in vp.Multiples.Values)
{
values.Add(vsp);
}
}
valueMap.Add(new StringField(propertyName), new CollectionField(values));
}
}
else
{
foreach (VertexPropertyField property in vertexField.VertexProperties.Values)
{
string propertyName = property.PropertyName;
Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName));
Debug.Assert(!propertyName.Equals(KW_VERTEX_EDGE));
Debug.Assert(!propertyName.Equals(KW_VERTEX_REV_EDGE));
switch (propertyName)
{
case "_rid":
case "_self":
case "_etag":
case "_attachments":
case "_ts":
continue;
default:
List <FieldObject> values = new List <FieldObject>();
foreach (VertexSinglePropertyField singleVp in property.Multiples.Values)
{
values.Add(singleVp);
}
valueMap.Add(new StringField(propertyName), new CollectionField(values));
break;
}
}
}
}
else if (inputTarget is EdgeField)
{
EdgeField edgeField = (EdgeField)inputTarget;
if (this.propertyNameList.Any())
{
foreach (string propertyName in this.propertyNameList)
{
FieldObject property = edgeField[propertyName];
if (property == null)
{
continue;
}
EdgePropertyField edgePf = property as EdgePropertyField;
if (edgePf != null)
{
valueMap.Add(new StringField(propertyName), edgePf);
}
}
}
else
{
foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties)
{
string propertyName = propertyPair.Key;
EdgePropertyField edgePropertyField = propertyPair.Value;
switch (propertyName)
{
// Reserved properties for meta-data
case KW_EDGE_ID:
//case KW_EDGE_OFFSET:
case KW_EDGE_SRCV:
case KW_EDGE_SINKV:
case KW_EDGE_SRCV_LABEL:
case KW_EDGE_SINKV_LABEL:
continue;
default:
valueMap.Add(new StringField(propertyName), edgePropertyField);
break;
}
}
}
}
else if (inputTarget is VertexSinglePropertyField)
{
VertexSinglePropertyField singleVp = inputTarget as VertexSinglePropertyField;
if (this.propertyNameList.Any())
{
foreach (string propertyName in this.propertyNameList)
{
FieldObject property = singleVp[propertyName];
if (property == null)
{
continue;
}
ValuePropertyField metaPf = property as ValuePropertyField;
if (metaPf != null)
{
valueMap.Add(new StringField(propertyName), metaPf);
}
}
}
else
{
foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties)
{
valueMap.Add(new StringField(kvp.Key), kvp.Value);
}
}
}
else
{
throw new GraphViewException("The input of valueMap() cannot be a meta or edge property.");
}
RawRecord result = new RawRecord();
result.Append(valueMap);
return(new List <RawRecord> {
result
});
}
public IEnumerator <Tuple <VertexField, RawRecord> > GetVerticesAndEdgesViaVertices(JsonQuery vertexQuery,
GraphViewCommand command)
{
IEnumerable <JObject> items = this.ExecuteQueryScript(vertexQuery);
List <string> nodeProperties = new List <string>(vertexQuery.NodeProperties);
List <string> edgeProperties = new List <string>(vertexQuery.EdgeProperties);
string nodeAlias = nodeProperties[0];
// Skip i = 0, which is the (node.* as nodeAlias) field
nodeProperties.RemoveAt(0);
//
// TODO: Refactor
//
string edgeAlias = null;
bool isReverseAdj = false;
bool isStartVertexTheOriginVertex = false;
bool crossApplyEdgeOnServer = edgeProperties.Any();
if (crossApplyEdgeOnServer)
{
edgeAlias = edgeProperties[0];
isReverseAdj = bool.Parse(edgeProperties[1]);
isStartVertexTheOriginVertex = bool.Parse(edgeProperties[2]);
edgeProperties.RemoveAt(0);
edgeProperties.RemoveAt(0);
edgeProperties.RemoveAt(0);
}
//
// Batch strategy:
// - For "small" vertexes, they have been cross applied on the server side
// - For "large" vertexes, just return the VertexField, the adjacency list decoder will
// construct spilled adjacency lists in batch mode and cross apply edges after that
//
Func <VertexField, string, Tuple <VertexField, RawRecord> > makeCrossAppliedRecord = (vertexField, edgeId) => {
Debug.Assert(vertexField != null);
RawRecord nodeRecord = new RawRecord();
//
// Fill node property field
//
foreach (string propertyName in nodeProperties)
{
FieldObject propertyValue = vertexField[propertyName];
nodeRecord.Append(propertyValue);
}
RawRecord edgeRecord = new RawRecord(edgeProperties.Count);
EdgeField edgeField =
((AdjacencyListField)vertexField[isReverseAdj ? DocumentDBKeywords.KW_VERTEX_REV_EDGE : DocumentDBKeywords.KW_VERTEX_EDGE])
.GetEdgeField(edgeId, true);
string startVertexId = vertexField.VertexId;
AdjacencyListDecoder.FillMetaField(edgeRecord, edgeField, startVertexId, vertexField.Partition, isStartVertexTheOriginVertex, isReverseAdj);
AdjacencyListDecoder.FillPropertyField(edgeRecord, edgeField, edgeProperties);
nodeRecord.Append(edgeRecord);
return(new Tuple <VertexField, RawRecord>(vertexField, nodeRecord));
};
Func <VertexField, Tuple <VertexField, RawRecord> > makeRawRecord = (vertexField) => {
Debug.Assert(vertexField != null);
RawRecord rawRecord = new RawRecord();
//
// Fill node property field
//
foreach (string propertyName in nodeProperties)
{
FieldObject propertyValue = vertexField[propertyName];
rawRecord.Append(propertyValue);
}
return(new Tuple <VertexField, RawRecord>(vertexField, rawRecord));
};
HashSet <string> uniqueVertexIds = new HashSet <string>();
HashSet <string> uniqueEdgeIds = new HashSet <string>();
foreach (JObject dynamicItem in items)
{
JObject tmpVertexObject = (JObject)((JObject)dynamicItem)[nodeAlias];
string vertexId = (string)tmpVertexObject[DocumentDBKeywords.KW_DOC_ID];
if (crossApplyEdgeOnServer)
{
// Note: since vertex properties can be multi-valued,
// a DocumentDB query needs a join clause in the FROM clause
// to retrieve vertex property values, which may result in
// the same vertex being returned multiple times.
// We use the hash set uniqueVertexIds to ensure one vertex is
// produced only once.
if (EdgeDocumentHelper.IsBuildingTheAdjacencyListLazily(
tmpVertexObject,
isReverseAdj,
this.Connection.UseReverseEdges) &&
uniqueVertexIds.Add(vertexId))
{
VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject);
yield return(makeRawRecord(vertexField));
}
else // When the DocumentDB query crosses apply edges
{
JObject edgeObjct = (JObject)((JObject)dynamicItem)[edgeAlias];
string edgeId = (string)edgeObjct[DocumentDBKeywords.KW_EDGE_ID];
if (uniqueEdgeIds.Add(edgeId))
{
VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject);
yield return(makeCrossAppliedRecord(vertexField, edgeId));
}
}
}
else
{
if (!uniqueVertexIds.Add(vertexId))
{
continue;
}
VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, tmpVertexObject);
yield return(makeRawRecord(vertexField));
}
}
}
public IEnumerator <RawRecord> GetVerticesAndEdgesViaEdges(JsonQuery edgeQuery, GraphViewCommand command)
{
IEnumerable <JObject> items = this.ExecuteQueryScript(edgeQuery);
List <string> nodeProperties = new List <string>(edgeQuery.NodeProperties);
List <string> edgeProperties = new List <string>(edgeQuery.EdgeProperties);
string nodeAlias = nodeProperties[0];
nodeProperties.RemoveAt(0);
string edgeAlias = edgeProperties[0];
edgeProperties.RemoveAt(0);
HashSet <string> spilledVertexIdSet = new HashSet <string>();
HashSet <string> spilledVertexPartitionSet = new HashSet <string>();
//
// <vertex id, edge id>
//
Dictionary <string, List <string> > vertexIdAndEdgeIdsDict = new Dictionary <string, List <string> >();
//
// <vertex id, <edgeDocumentId, edgeObject>>
//
Dictionary <string, List <Tuple <string, JObject> > > vertexIdAndEdgeObjectsDict =
new Dictionary <string, List <Tuple <string, JObject> > >();
foreach (JObject dynamicItem in items)
{
JObject tmpObject = (JObject)dynamicItem[nodeAlias];
JObject edgeObject = (JObject)dynamicItem[edgeAlias];
//
// This is a spilled edge document
//
if (tmpObject[DocumentDBKeywords.KW_EDGEDOC_VERTEXID] != null)
{
string vertexId = tmpObject[DocumentDBKeywords.KW_EDGEDOC_VERTEXID].ToString();
spilledVertexIdSet.Add(vertexId);
string partition = this.Connection.GetDocumentPartition(tmpObject);
if (partition != null)
{
spilledVertexPartitionSet.Add(partition);
}
List <Tuple <string, JObject> > edgeObjects;
if (!vertexIdAndEdgeObjectsDict.TryGetValue(vertexId, out edgeObjects))
{
edgeObjects = new List <Tuple <string, JObject> >();
vertexIdAndEdgeObjectsDict.Add(vertexId, edgeObjects);
}
edgeObjects.Add(new Tuple <string, JObject>((string)tmpObject[DocumentDBKeywords.KW_DOC_ID], edgeObject));
List <string> edgeIds;
if (!vertexIdAndEdgeIdsDict.TryGetValue(vertexId, out edgeIds))
{
edgeIds = new List <string>();
vertexIdAndEdgeIdsDict.Add(vertexId, edgeIds);
}
edgeIds.Add((string)edgeObject[DocumentDBKeywords.KW_DOC_ID]);
}
else
{
string vertexId = (string)tmpObject[DocumentDBKeywords.KW_DOC_ID];
command.VertexCache.AddOrUpdateVertexField(vertexId, tmpObject);
List <string> edgeIds;
if (!vertexIdAndEdgeIdsDict.TryGetValue(vertexId, out edgeIds))
{
edgeIds = new List <string>();
vertexIdAndEdgeIdsDict.Add(vertexId, edgeIds);
}
edgeIds.Add((string)edgeObject[DocumentDBKeywords.KW_DOC_ID]);
}
}
if (spilledVertexIdSet.Any())
{
const string NODE_ALISE = "Node";
JsonQuery query = new JsonQuery
{
NodeAlias = NODE_ALISE
};
query.AddSelectElement("*");
query.RawWhereClause = new WInPredicate(new WColumnReferenceExpression(NODE_ALISE, "id"), spilledVertexIdSet.ToList());
if (spilledVertexPartitionSet.Any())
{
query.WhereConjunction(
new WInPredicate(
new WValueExpression($"{NODE_ALISE}{this.Connection.GetPartitionPathIndexer()}"),
spilledVertexPartitionSet.ToList()), BooleanBinaryExpressionType.And);
}
// TODO: remove this code when you understand it.
// string idInClause = string.Join(", ", spilledVertexIdSet.Select(id => $"'{id}'"));
// string partitionInClause = string.Join(", ", spilledVertexPartitionSet.Select(partition => $"'{partition}'"));
// string queryScript = $"SELECT * FROM Node WHERE Node.id IN ({idInClause})" +
// (string.IsNullOrEmpty(partitionInClause)
// ? ""
// : $" AND Node{this.Connection.GetPartitionPathIndexer()} IN ({partitionInClause})");
// IEnumerable<dynamic> spilledVertices = this.Connection.ExecuteDocDbQuery(queryScript);
IEnumerable <JObject> spilledVertices = this.ExecuteQueryScript(query);
foreach (JObject vertex in spilledVertices)
{
JObject vertexObject = vertex;
string vertexId = (string)vertexObject[DocumentDBKeywords.KW_DOC_ID];
VertexField vertexField = command.VertexCache.AddOrUpdateVertexField(vertexId, vertexObject);
vertexField.ConstructPartialLazyAdjacencyList(vertexIdAndEdgeObjectsDict[vertexId], false);
}
}
foreach (KeyValuePair <string, List <string> > pair in vertexIdAndEdgeIdsDict)
{
string vertexId = pair.Key;
List <string> edgeIds = pair.Value;
VertexField vertexField = command.VertexCache.GetVertexField(vertexId);
foreach (string edgeId in edgeIds)
{
RawRecord nodeRecord = new RawRecord();
//
// Fill node property field
//
foreach (string propertyName in nodeProperties)
{
FieldObject propertyValue = vertexField[propertyName];
nodeRecord.Append(propertyValue);
}
RawRecord edgeRecord = new RawRecord(edgeProperties.Count);
EdgeField edgeField = vertexField.AdjacencyList.GetEdgeField(edgeId, false);
Debug.Assert(edgeField != null, "edgeField != null");
string startVertexId = vertexField.VertexId;
AdjacencyListDecoder.FillMetaField(edgeRecord, edgeField, startVertexId, vertexField.Partition, true, false);
AdjacencyListDecoder.FillPropertyField(edgeRecord, edgeField, edgeProperties);
nodeRecord.Append(edgeRecord);
yield return(nodeRecord);
}
}
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
var results = new List <RawRecord>();
// Extract all properties if allPropertyIndex >= 0
if (allPropertyIndex >= 0 && record[allPropertyIndex] != null)
{
VertexField vertexField = record[allPropertyIndex] as VertexField;
if (vertexField != null)
{
foreach (PropertyField property in vertexField.AllProperties)
{
string propertyName = property.PropertyName;
switch (propertyName)
{
// Reversed properties for meta-data
case "_edge":
case "_partition":
case "_reverse_edge":
case "_nextEdgeOffset":
case "_rid":
case "_self":
case "_etag":
case "_attachments":
case "_ts":
continue;
default:
RawRecord r = new RawRecord();
if (property is VertexSinglePropertyField || property is ValuePropertyField)
{
r.Append(property);
}
else if (property is VertexPropertyField)
{
foreach (VertexSinglePropertyField p in ((VertexPropertyField)property).Multiples.Values)
{
r.Append(p);
}
}
else
{
Debug.Assert(false, $"[PropertiesOperator.CrossApply] property type error: {property.GetType()}");
}
results.Add(r);
break;
}
}
}
else
{
EdgeField edgeField = record[allPropertyIndex] as EdgeField;
if (edgeField == null)
{
throw new GraphViewException(
string.Format("The FieldObject record[{0}] should be a VertexField or EdgeField but now it is {1}.",
allPropertyIndex, record[allPropertyIndex].ToString()));
}
foreach (var propertyPair in edgeField.EdgeProperties)
{
string propertyName = propertyPair.Key;
EdgePropertyField propertyField = propertyPair.Value;
switch (propertyName)
{
// Reversed properties for meta-data
case "_offset":
case "_srcV":
case "_sinkV":
case "_srcVLabel":
case "_sinkVLabel":
continue;
default:
RawRecord r = new RawRecord();
r.Append(propertyField);
results.Add(r);
break;
}
}
}
}
else
{
// TODO: Now translation code needn't to generate the key name for the operator
foreach (var pair in propertyList)
{
//string propertyName = pair.Item1;
int propertyValueIndex = pair.Item2;
var propertyValue = record[propertyValueIndex];
if (propertyValue == null)
{
continue;
}
var result = new RawRecord();
result.Append(propertyValue);
results.Add(result);
}
}
return(results);
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
var results = new List <RawRecord>();
// Extract all values if allValuesIndex >= 0
if (allValuesIndex >= 0 && record[allValuesIndex] != null)
{
VertexField vertexField = record[allValuesIndex] as VertexField;
if (vertexField != null)
{
foreach (PropertyField property in vertexField.AllProperties)
{
string propertyName = property.PropertyName;
switch (propertyName)
{
// Reversed properties for meta-data
case "_edge":
case "_reverse_edge":
case "_partition":
case "_nextEdgeOffset":
case "_rid":
case "_self":
case "_etag":
case "_attachments":
case "_ts":
continue;
default:
RawRecord r = new RawRecord();
r.Append(new StringField(property.PropertyValue, property.JsonDataType));
results.Add(r);
break;
}
}
}
else
{
EdgeField edgeField = record[allValuesIndex] as EdgeField;
if (edgeField == null)
{
throw new GraphViewException(
string.Format("The FieldObject record[{0}] should be a VertexField or EdgeField but now it is {1}.",
allValuesIndex, record[allValuesIndex].ToString()));
}
foreach (var propertyPair in edgeField.EdgeProperties)
{
string propertyName = propertyPair.Key;
EdgePropertyField propertyField = propertyPair.Value;
switch (propertyName)
{
// Reversed properties for meta-data
case "_offset":
case "_srcV":
case "_srcVLabel":
case "_sinkV":
case "_sinkVLabel":
continue;
default:
RawRecord r = new RawRecord();
r.Append(new StringField(propertyField.ToValue, propertyField.JsonDataType));
results.Add(r);
break;
}
}
}
}
else
{
foreach (var propIdx in ValuesIdxList)
{
PropertyField propertyValue = record[propIdx] as PropertyField;
if (propertyValue == null)
{
continue;
}
var result = new RawRecord();
result.Append(new StringField(propertyValue.ToValue, propertyValue.JsonDataType));
results.Add(result);
}
}
return(results);
}
internal override List <RawRecord> CrossApply(RawRecord record)
{
List <RawRecord> results = new List <RawRecord>();
FieldObject inputTarget = record[this.inputTargetIndex];
if (inputTarget is VertexField)
{
VertexField vertexField = (VertexField)inputTarget;
foreach (VertexPropertyField property in vertexField.VertexProperties.Values)
{
string propertyName = property.PropertyName;
Debug.Assert(!VertexField.IsVertexMetaProperty(propertyName));
Debug.Assert(propertyName == "_edge");
Debug.Assert(propertyName == "_reverse_edge");
switch (propertyName)
{
case "_rid":
case "_self":
case "_etag":
case "_attachments":
case "_ts":
continue;
default:
foreach (VertexSinglePropertyField singleVp in property.Multiples.Values)
{
RawRecord r = new RawRecord();
r.Append(new StringField(singleVp.PropertyValue, singleVp.JsonDataType));
results.Add(r);
}
break;
}
}
}
else if (inputTarget is EdgeField)
{
EdgeField edgeField = (EdgeField)inputTarget;
foreach (KeyValuePair <string, EdgePropertyField> propertyPair in edgeField.EdgeProperties)
{
string propertyName = propertyPair.Key;
EdgePropertyField edgePropertyField = propertyPair.Value;
switch (propertyName)
{
// Reserved properties for meta-data
case "_edgeId":
case "_offset":
case "_srcV":
case "_sinkV":
case "_srcVLabel":
case "_sinkVLabel":
continue;
default:
RawRecord r = new RawRecord();
r.Append(new StringField(edgePropertyField.PropertyValue, edgePropertyField.JsonDataType));
results.Add(r);
break;
}
}
}
else if (inputTarget is VertexSinglePropertyField)
{
VertexSinglePropertyField singleVp = inputTarget as VertexSinglePropertyField;
foreach (KeyValuePair <string, ValuePropertyField> kvp in singleVp.MetaProperties)
{
RawRecord r = new RawRecord();
ValuePropertyField metaPropertyField = kvp.Value;
r.Append(new StringField(metaPropertyField.PropertyValue, metaPropertyField.JsonDataType));
results.Add(r);
}
}
else
{
throw new GraphViewException("The input of values() cannot be a meta or edge property.");
}
return(results);
}
internal override RawRecord DataModify(RawRecord record)
{
long edgeOffset = long.Parse(record[this._edgeOffsetIndex].ToValue);
string srcVertexId = record[this._srcVertexIdIndex].ToValue;
JObject srcVertexObject = this.Connection.RetrieveDocumentById(srcVertexId);
string outEdgeDocId;
JObject outEdgeObject;
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, srcVertexObject, srcVertexId, edgeOffset, false,
out outEdgeObject, out outEdgeDocId);
if (outEdgeObject == null)
{
// TODO: Is there something wrong?
Debug.WriteLine($"[UpdateEdgePropertiesOperator] The edge does not exist: vertexId = {srcVertexId}, edgeOffset = {edgeOffset}");
return(null);
}
string sinkVertexId = (string)outEdgeObject["_sinkV"];
JObject sinkVertexObject;
string inEdgeDocId;
JObject inEdgeObject;
if (sinkVertexId.Equals(srcVertexId))
{
sinkVertexObject = srcVertexObject; // NOTE: Must not use DeepClone() here!
}
else
{
sinkVertexObject = this.Connection.RetrieveDocumentById(sinkVertexId);
}
EdgeDocumentHelper.FindEdgeBySourceAndOffset(
this.Connection, sinkVertexObject, srcVertexId, edgeOffset, true,
out inEdgeObject, out inEdgeDocId);
VertexField srcVertexField = this.Connection.VertexCache.GetVertexField(srcVertexId);
VertexField sinkVertexField = this.Connection.VertexCache.GetVertexField(sinkVertexId);
EdgeField outEdgeField = srcVertexField.AdjacencyList.GetEdgeField(srcVertexId, edgeOffset);
EdgeField inEdgeField = sinkVertexField.RevAdjacencyList.GetEdgeField(srcVertexId, edgeOffset);
// Drop all non-reserved properties
if (this.PropertiesToBeUpdated.Count == 1 &&
!this.PropertiesToBeUpdated[0].Item1.SingleQuoted &&
this.PropertiesToBeUpdated[0].Item1.Value.Equals("*", StringComparison.OrdinalIgnoreCase) &&
!this.PropertiesToBeUpdated[0].Item2.SingleQuoted &&
this.PropertiesToBeUpdated[0].Item2.Value.Equals("null", StringComparison.OrdinalIgnoreCase))
{
List <string> toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(outEdgeObject);
foreach (var propertyName in toBeDroppedProperties)
{
outEdgeField.EdgeProperties.Remove(propertyName);
}
toBeDroppedProperties = GraphViewJsonCommand.DropAllEdgeProperties(inEdgeObject);
foreach (var propertyName in toBeDroppedProperties)
{
inEdgeField.EdgeProperties.Remove(propertyName);
}
}
else
{
foreach (Tuple <WValueExpression, WValueExpression, int> tuple in this.PropertiesToBeUpdated)
{
WValueExpression keyExpression = tuple.Item1;
WValueExpression valueExpression = tuple.Item2;
if (this.Mode == UpdatePropertyMode.Set)
{
// Modify edgeObject (update the edge property)
JProperty updatedProperty = GraphViewJsonCommand.UpdateProperty(outEdgeObject, keyExpression, valueExpression);
// Update VertexCache
if (updatedProperty == null)
{
outEdgeField.EdgeProperties.Remove(keyExpression.Value);
}
else
{
outEdgeField.UpdateEdgeProperty(updatedProperty, outEdgeField);
}
// Modify edgeObject (update the edge property)
updatedProperty = GraphViewJsonCommand.UpdateProperty(inEdgeObject, keyExpression, valueExpression);
// Update VertexCache
if (updatedProperty == null)
{
inEdgeField.EdgeProperties.Remove(keyExpression.Value);
}
else
{
inEdgeField.UpdateEdgeProperty(updatedProperty, inEdgeField);
}
}
else
{
throw new NotImplementedException();
}
}
}
// Interact with DocDB to update the property
EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, srcVertexObject, outEdgeDocId, false, outEdgeObject);
EdgeDocumentHelper.UpdateEdgeProperty(this.Connection, sinkVertexObject, inEdgeDocId, true, inEdgeObject);
//
// Drop edge property
//
if (this.PropertiesToBeUpdated.Any(t => t.Item2 == null))
{
return(null);
}
return(record);
}
