protected override Delegate GetGetterCore(IChannel ch, Row input, int iinfo, out Action disposer)
{
Host.AssertValueOrNull(ch);
Host.AssertValue(input);
Host.Assert(0 <= iinfo && iinfo < Infos.Length);
var type = _types[iinfo];
var ex = _exes[iinfo];
bool needScale = ex.Offset != 0 || ex.Scale != 1;
disposer = null;
var sourceType = InputSchema.GetColumnType(Infos[iinfo].Source);
if (sourceType.ItemType == NumberType.R4 || sourceType.ItemType == NumberType.R8)
{
return(GetterFromType <float>(input, iinfo, ex, needScale));
}
else
if (sourceType.ItemType == NumberType.U1)
{
return(GetterFromType <byte>(input, iinfo, ex, false));
}
else
{
throw Contracts.Except("We only support float or byte arrays");
}
}
private static object CreateEvent(string topicName, InputSchema inputSchema)
{
Random random = new Random();
switch (inputSchema)
{
case InputSchema.EventGridSchema:
string subject = $"sensor:{random.Next(1, 100)}";
double temperature = random.NextDouble();
double pressure = random.NextDouble();
double humidity = random.Next(1, 25);
return(new EventGridEvent()
{
Id = Guid.NewGuid().ToString(),
Topic = topicName,
Subject = subject,
EventType = "sensor.temperature",
DataVersion = "1.0",
EventTime = DateTime.UtcNow,
Data = new
{
Machine = new { Temperature = temperature, Pressure = pressure },
Ambient = new { Temperature = temperature, Humidity = humidity },
},
});
default:
throw new NotImplementedException();
}
}
internal override void PropagateDataTypesForInSchema()
{
// Input for JOIN is the combination of both sources
var prevLeftSchema = InOperatorLeft.OutputSchema;
var prevRightSchema = InOperatorRight.OutputSchema;
var prevOutFields = prevLeftSchema.Union(prevRightSchema);
var fieldMapping = new Dictionary <Field, Field>();
foreach (var inF in InputSchema)
{
var matchOutFields = prevOutFields.Where(f => f.FieldAlias == inF.FieldAlias);
if (matchOutFields.Count() > 1)
{
// In case of join by node ids between left and right input (such as in MATCH .. OPTIONAL MATCH ... WHERE case)
// we will take the from left side and ignore duplications
// otherwise throws exception
if (!JoinPairs.All(jp => jp.Type == JoinKeyPair.JoinKeyPairType.NodeId))
{
throw new TranspilerInternalErrorException($"Ambiguous match of field with alias '{inF.FieldAlias}'");
}
}
if (matchOutFields.Count() == 0)
{
throw new TranspilerInternalErrorException($"Failed to match field with alias '{inF.FieldAlias}'");
}
fieldMapping.Add(inF, matchOutFields.First());
}
CopyFieldInfoHelper(InputSchema, fieldMapping.Values);
// additional validation on if relationships can satisfy the joins if directional traversal is explicitly specified
foreach (var joinPair in JoinPairs)
{
var nodeEntity = InputSchema.First(s => s.FieldAlias == joinPair.NodeAlias) as EntityField;
var relEntity = InputSchema.First(s => s.FieldAlias == joinPair.RelationshipOrNodeAlias) as EntityField;
switch (joinPair.Type)
{
case JoinKeyPair.JoinKeyPairType.Source:
if (relEntity.BoundSourceEntityName != nodeEntity.BoundEntityName)
{
throw new TranspilerBindingException($"Cannot find relationship ({nodeEntity.BoundEntityName})-[{relEntity.BoundSourceEntityName}]->(*), for {joinPair.NodeAlias} and {joinPair.RelationshipOrNodeAlias}");
}
break;
case JoinKeyPair.JoinKeyPairType.Sink:
if (relEntity.BoundSinkEntityName != nodeEntity.BoundEntityName)
{
throw new TranspilerBindingException($"Cannot find relationship ({nodeEntity.BoundEntityName})<-[{relEntity.BoundSourceEntityName}]-(*), for {joinPair.NodeAlias} and {joinPair.RelationshipOrNodeAlias}");
}
break;
default:
// in .Either, or .Both case
// no validation need to be done as Binding should already enforce the existence of the relationships
break;
}
}
}
internal override void PropagateDataTypesForOutSchema()
{
// Output for JOIN depends on type of the join
// For LEFT join, attributes becomes non-nullable
// For CROSS join or INNER join, attributes type remains unchanged after join
if (Type == JoinType.Left)
{
var prevLeftSchema = InOperatorLeft.OutputSchema;
var prevRightSchema = InOperatorRight.OutputSchema;
var fieldMapping = new Dictionary <Field, Field>();
foreach (var outField in OutputSchema)
{
var inFieldMatches = InputSchema.Where(f => f.FieldAlias == outField.FieldAlias);
Debug.Assert(InputSchema.Where(f => f.FieldAlias == outField.FieldAlias).Count() == 1); // must have match in IN for any OUT field
var inField = inFieldMatches.First();
// we made it that left schema LEFT OUTTER JOIN with right schema always
var isFromRight = !prevLeftSchema.Any(f => f.FieldAlias == inField.FieldAlias);
// copy over the schema first
outField.Copy(inField);
// make adjustment for outter join
if (inField is ValueField)
{
Debug.Assert(outField.GetType() == inField.GetType()); // join doesn't alter field types
var outFieldSingleField = outField as ValueField;
if (isFromRight && !TypeHelper.CanAssignNullToType(outFieldSingleField.FieldType))
{
// make it nullable variant of the original type
outFieldSingleField.FieldType = TypeHelper.GetNullableTypeForType(outFieldSingleField.FieldType);
}
}
else
{
Debug.Assert(outField is EntityField);
var outEntCapFields = (outField as EntityField).EncapsulatedFields;
if (isFromRight)
{
foreach (var outEntCapField in outEntCapFields)
{
if (!TypeHelper.CanAssignNullToType(outEntCapField.FieldType))
{
// make it nullable variant of the original type
outEntCapField.FieldType = TypeHelper.GetNullableTypeForType(outEntCapField.FieldType);
}
}
}
}
}
}
else
{
base.PropagateDataTypesForOutSchema();
}
}
/// <summary>
/// Given a set of columns, return the input columns that are needed to generate those output columns.
/// </summary>
IEnumerable <DataViewSchema.Column> ISchemaBoundRowMapper.GetDependenciesForNewColumns(IEnumerable <DataViewSchema.Column> dependingColumns)
{
if (dependingColumns.Count() == 0)
{
return(Enumerable.Empty <DataViewSchema.Column>());
}
return(InputSchema.Where(col => _inputColIndices.Contains(col.Index)));
}
/// <summary>
/// Used by logic planner to update the list of ReferencedFields
/// </summary>
internal virtual void PropagateReferencedPropertiesForEntityFields()
{
// lift the referenced fields from the next layer of operators back to this one's output entity fields
var referredFieldsInDownstreamOps = OutOperators.SelectMany(op => op.InputSchema)
.Where(f => f is EntityField)
.Cast <EntityField>()
.GroupBy(f => f.FieldAlias)
.ToDictionary(kv => kv.Key, kv => kv.SelectMany(fn => fn.ReferencedFieldAliases).Distinct().ToList());
foreach (var field in OutputSchema.Where(f => f is EntityField).Cast <EntityField>())
{
Debug.Assert(referredFieldsInDownstreamOps.ContainsKey(field.FieldAlias));
field.AddReferenceFieldNames(referredFieldsInDownstreamOps[field.FieldAlias]);
}
// lift the referenced fields in the entity fields from output to input schema of this operator, if
// applicable
if ((InputSchema?.Count ?? 0) > 0)
{
// handle entity name renamed cases by creating a map from field alias before projection to after
// projection
var aliasMap = new Dictionary <String, String>();
if (this is ProjectionOperator)
{
var aliasMap1 = (this as ProjectionOperator).ProjectionMap
.Where(u => OutputSchema.Where(n => n is EntityField).Any(k => k.FieldAlias == u.Key));
aliasMap = aliasMap1?.ToDictionary(n => n.Value.GetChildrenQueryExpressionType <QueryExpressionProperty>().First().VariableName, n => n.Key);
}
else
{
// create a dummy alias map ( A -> A, B -> B; not alias name modified) for non-projection operator
aliasMap = OutputSchema.Where(n => n is EntityField).ToDictionary(n => n.FieldAlias, n => n.FieldAlias);
}
foreach (var field in InputSchema.Where(f => f is EntityField).Cast <EntityField>())
{
var mappedAlias = (aliasMap.ContainsKey(field.FieldAlias) ?aliasMap[field.FieldAlias]:null);
if (mappedAlias != null && referredFieldsInDownstreamOps.ContainsKey(mappedAlias))
{
field.AddReferenceFieldNames(referredFieldsInDownstreamOps[mappedAlias]);
}
}
var referredFieldsForUpstream = InputSchema
.Where(f => f is EntityField).Cast <EntityField>()
.ToDictionary(kv => kv.FieldAlias, kv => kv);
// Some operators has additional fields may get referenced even they are not in output schema
// Such as in WHERE or ORDER BY
// Child logical operator class implement this and does the appending
AppendReferencedProperties(referredFieldsForUpstream);
}
}
protected override Schema.DetachedColumn[] GetOutputColumnsCore()
{
var result = new Schema.DetachedColumn[_parent.ColumnPairs.Length];
for (int i = 0; i < _parent.ColumnPairs.Length; i++)
{
InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].input, out int colIndex);
Host.Assert(colIndex >= 0);
result[i] = new Schema.DetachedColumn(_parent.ColumnPairs[i].output, _types[i], null);
}
return result;
}
internal void UpdatePropertyBasedOnAliasFromInputSchema(QueryExpressionProperty prop)
{
var matchedField = InputSchema.FirstOrDefault(f => f.FieldAlias == prop.VariableName);
if (matchedField == null)
{
throw new TranspilerBindingException($"Alias '{prop.VariableName}' does not exist in the current context");
}
if (string.IsNullOrEmpty(prop.PropertyName))
{
// direct reference to an alias (value column or entity column)
if (matchedField is ValueField)
{
prop.DataType = (matchedField as ValueField).FieldType;
}
else
{
// entity field reference in a single field expression
// this is valid only in handful situations, such as Count(d), Count(distinct(d))
// in such case, we populate the Entity object with correct entity type so that code generator can use it later
Debug.Assert(matchedField is EntityField);
var matchedEntity = matchedField as EntityField;
prop.Entity = matchedEntity.Type == EntityField.EntityType.Node ?
new NodeEntity()
{
EntityName = matchedEntity.EntityName, Alias = matchedEntity.FieldAlias
} as Entity :
new RelationshipEntity()
{
EntityName = matchedEntity.EntityName, Alias = matchedEntity.FieldAlias
} as Entity;
}
}
else
{
// property dereference of an entity column
if (!(matchedField is EntityField))
{
throw new TranspilerBindingException($"Failed to dereference property {prop.PropertyName} for alias {prop.VariableName}, which is not an alias of entity type as expected");
}
var entField = matchedField as EntityField;
var entPropField = entField.EncapsulatedFields.FirstOrDefault(f => f.FieldAlias == prop.PropertyName);
if (entPropField == null)
{
throw new TranspilerBindingException($"Failed to dereference property {prop.PropertyName} for alias {prop.VariableName}, Entity type {entField.BoundEntityName} does not have a property named {prop.PropertyName}");
}
entField.AddReferenceFieldName(prop.PropertyName);
prop.DataType = entPropField.FieldType;
}
}
/// <summary>
/// For PCA, the transform equation is y=U^Tx, where "^T" denotes matrix transpose, x is an 1-D vector (i.e., the input column), and U=[u_1, ..., u_PcaNum]
/// is a n-by-PcaNum matrix. The symbol u_k is the k-th largest (in terms of the associated eigenvalue) eigenvector of (1/m)*\sum_{i=1}^m x_ix_i^T,
/// where x_i is the whitened column at the i-th row and we have m rows in the training data.
/// For ZCA, the transform equation is y = US^{-1/2}U^Tx, where U=[u_1, ..., u_n] (we retain all eigenvectors) and S is a diagonal matrix whose i-th
/// diagonal element is the eigenvalues of u_i. The first U^Tx rotates x to another linear space (bases are u_1, ..., u_n), then S^{-1/2} is applied
/// to ensure unit variance, and finally we rotate the scaled result back to the original space using U (note that UU^T is identity matrix so U is
/// the inverse rotation of U^T).
/// </summary>
protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
{
var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length];
for (int iinfo = 0; iinfo < _parent.ColumnPairs.Length; iinfo++)
{
InputSchema.TryGetColumnIndex(_parent.ColumnPairs[iinfo].inputColumnName, out int colIndex);
Host.Assert(colIndex >= 0);
var info = _parent._columns[iinfo];
DataViewType outType = (info.Kind == WhiteningKind.PrincipalComponentAnalysis && info.Rank > 0) ? new VectorDataViewType(NumberDataViewType.Single, info.Rank) : _srcTypes[iinfo];
result[iinfo] = new DataViewSchema.DetachedColumn(_parent.ColumnPairs[iinfo].outputColumnName, outType, null);
}
return(result);
}
/// <summary>
/// For PCA, the transform equation is y=U^Tx, where "^T" denotes matrix transpose, x is an 1-D vector (i.e., the input column), and U=[u_1, ..., u_PcaNum]
/// is a n-by-PcaNum matrix. The symbol u_k is the k-th largest (in terms of the associated eigenvalue) eigenvector of (1/m)*\sum_{i=1}^m x_ix_i^T,
/// where x_i is the whitened column at the i-th row and we have m rows in the training data.
/// For ZCA, the transform equation is y = US^{-1/2}U^Tx, where U=[u_1, ..., u_n] (we retain all eigenvectors) and S is a diagonal matrix whose i-th
/// diagonal element is the eigenvalues of u_i. The first U^Tx rotates x to another linear space (bases are u_1, ..., u_n), then S^{-1/2} is applied
/// to ensure unit variance, and finally we rotate the scaled result back to the original space using U (note that UU^T is identity matrix so U is
/// the inverse rotation of U^T).
/// </summary>
protected override Schema.DetachedColumn[] GetOutputColumnsCore()
{
var result = new Schema.DetachedColumn[_parent.ColumnPairs.Length];
for (int iinfo = 0; iinfo < _parent.ColumnPairs.Length; iinfo++)
{
InputSchema.TryGetColumnIndex(_parent.ColumnPairs[iinfo].input, out int colIndex);
Host.Assert(colIndex >= 0);
var info = _parent._columns[iinfo];
ColumnType outType = (info.Kind == WhiteningKind.Pca && info.PcaNum > 0) ? new VectorType(NumberType.Float, info.PcaNum) : _srcTypes[iinfo];
result[iinfo] = new Schema.DetachedColumn(_parent.ColumnPairs[iinfo].output, outType, null);
}
return(result);
}
internal override void PropagateDateTypesForOutSchema()
{
// projection may alter the schema with calculated columns
// we calculate the data type of all the fields in the output schema
// using type evaluation method on the QueryExpression
// Map from out_alias to { projection_expression,
// e.g.: (a.b + c) AS d
// "d" -> (<expression of a.b+c>, <d's field in OutputSchema>)
var exprToOutputMap = ProjectionMap.ToDictionary(
kv => kv.Key, // key is output alias
kv => new { Expr = kv.Value, Field = OutputSchema.First(f => f.FieldAlias == kv.Key) } // value is the corresponding field object and expression
);
foreach (var map in exprToOutputMap)
{
// toggle the fact if any of the output column requires aggregation
HasAggregationField = HasAggregationField ||
(map.Value.Expr.GetChildrenQueryExpressionType <QueryExpressionAggregationFunction>().Count() > 0);
var allPropertyReferences = map.Value.Expr.GetChildrenQueryExpressionType <QueryExpressionProperty>();
// update types for all QueryExpressionPropty object first based on InputSchema (so QueryExpression.EvaluteType() will work)
foreach (var prop in allPropertyReferences)
{
UpdatePropertyBasedOnAliasFromInputSchema(prop);
}
// then, update the type in the OutputSchema
if (map.Value.Field is EntityField)
{
// This can only be direct exposure of entity (as opposed to deference of a particular property)
// We just copy of the fields that the entity can potentially be dereferenced
Debug.Assert(allPropertyReferences.Count() == 1);
var varName = allPropertyReferences.First().VariableName;
var matchInputField = InputSchema.First(f => f.FieldAlias == varName);
map.Value.Field.Copy(matchInputField);
}
else
{
// This can be a complex expression involve multiple field/column references
// We will compute the type of the expression
Debug.Assert(map.Value.Field is ValueField);
var evalutedType = map.Value.Expr.EvaluateType();
var outField = map.Value.Field as ValueField;
outField.FieldType = evalutedType;
}
}
}
private KeyToValueMap GetKeyMetadata <TKey, TValue>(int iinfo, ColumnType typeKey, ColumnType typeVal)
{
Host.Assert(0 <= iinfo && iinfo < _parent.ColumnPairs.Length);
Host.AssertValue(typeKey);
Host.AssertValue(typeVal);
Host.Assert(typeKey.ItemType.RawType == typeof(TKey));
Host.Assert(typeVal.ItemType.RawType == typeof(TValue));
var keyMetadata = default(VBuffer <TValue>);
InputSchema.GetMetadata(MetadataUtils.Kinds.KeyValues, ColMapNewToOld[iinfo], ref keyMetadata);
Host.Check(keyMetadata.Length == typeKey.ItemType.KeyCount);
VBufferUtils.Densify(ref keyMetadata);
return(new KeyToValueMap <TKey, TValue>(this, typeKey.ItemType.AsKey, typeVal.ItemType.AsPrimitive, keyMetadata.Values, iinfo));
}
protected override Schema.DetachedColumn[] GetOutputColumnsCore()
{
var result = new Schema.DetachedColumn[_parent.ColumnPairs.Length];
for (int iinfo = 0; iinfo < _infos.Length; iinfo++)
{
InputSchema.TryGetColumnIndex(_infos[iinfo].Input, out int colIndex);
Host.Assert(colIndex >= 0);
var builder = new MetadataBuilder();
builder.Add(InputSchema[colIndex].Metadata, x => x == MetadataUtils.Kinds.SlotNames);
ValueGetter <bool> getter = (ref bool dst) =>
{
dst = true;
};
builder.Add(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, getter);
result[iinfo] = new Schema.DetachedColumn(_infos[iinfo].Output, _infos[iinfo].OutputType, builder.GetMetadata());
}
return(result);
}
private static async Task WaitUntilEventGridModuleIsUpAndTopicExistsAsync(
GridConfiguration gridConfig,
EventGridEdgeClient egClient,
CancellationToken cancellationToken)
{
InputSchema inputSchema = GetTopicInputSchema(gridConfig);
Topic topic = new Topic
{
Name = gridConfig.Topic.Name,
Properties = new TopicProperties()
{
InputSchema = inputSchema,
},
};
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
using (CancellationTokenSource cts = new CancellationTokenSource(30 * 1000))
{
var createdTopic = await egClient.Topics.PutTopicAsync(topicName : topic.Name, topic : topic, cts.Token).ConfigureAwait(false);
Console.WriteLine($"Successfully created topic with name {topic.Name} so event grid must be up...");
break;
}
}
catch (EventGridApiException e)
{
LogAndBackoff(topic.Name, e);
}
catch (HttpRequestException e)
{
LogAndBackoff(topic.Name, e);
}
}
}
public Mapper(VectorWhiteningTransformer parent, Schema inputSchema)
: base(parent.Host.Register(nameof(Mapper)), parent, inputSchema)
{
_parent = parent;
_cols = new int[_parent.ColumnPairs.Length];
_srcTypes = new ColumnType[_parent.ColumnPairs.Length];
for (int i = 0; i < _parent.ColumnPairs.Length; i++)
{
if (!InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].input, out _cols[i]))
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _parent.ColumnPairs[i].input);
}
_srcTypes[i] = inputSchema[_cols[i]].Type;
ValidateModel(Host, _parent._models[i], _srcTypes[i]);
if (_parent._columns[i].SaveInv)
{
ValidateModel(Host, _parent._invModels[i], _srcTypes[i]);
}
}
}
public OutputSchema GetReduction(InputSchema requests)
{
var response = new OutputSchema();
requests.SellEntries = requests.BuyEntries.OrderBy(b => b.Value).ThenByDescending(b => b.IsMarket).ToList();
requests.BuyEntries = requests.BuyEntries.OrderByDescending(b => b.Value).ThenBy(b => b.IsMarket).ToList();
while (requests.SellEntries.Count() > 0 || requests.BuyEntries.Count() > 0)
{
var sell = requests.SellEntries.FirstOrDefault();
var buy = requests.BuyEntries.FirstOrDefault();
var output = new OutputEntry();
output.BuyOrderId = buy.Id;
output.SellOrderId = sell.Id;
if (sell.Volume > buy.Volume)
{
output.Value = buy.Value;
output.Volume = buy.Volume;
sell.Volume -= buy.Volume;
response.OutputEnties.Add(output);
requests.BuyEntries.Remove(buy);
}
else
{
output.Value = buy.Value;
output.Volume = sell.Volume;
buy.Volume -= sell.Volume;
response.OutputEnties.Add(output);
requests.SellEntries.Remove(sell);
}
}
response.IsOk = true;
return(response);
}
public static InputSchema DeserializeFile(string filePath)
{
var result = new InputSchema
{
BuyEntries = new System.Collections.Generic.List <InputEntry>(),
SellEntries = new System.Collections.Generic.List <InputEntry>()
};
using StreamReader sr = new StreamReader(filePath);
int id = 1;
while (!sr.EndOfStream)
{
var line = sr.ReadLine();
var array = line.Split(',');
var entry = new InputEntry
{
Id = id,
IsBuy = true,
IsMarket = array[1].Trim().ToLower() == "m",
Volume = long.Parse(array[2].Trim().ToLower())
};
entry.Value = entry.IsMarket ? -1 : long.Parse(array[3].Trim().ToLower());
if (entry.IsBuy)
{
result.BuyEntries.Add(entry);
}
else
{
result.SellEntries.Add(entry);
}
id++;
}
return(result);
}
public RowMapper(IHostEnvironment env, BindableMapper parent, RoleMappedSchema schema)
{
Contracts.AssertValue(env);
_env = env;
_env.AssertValue(schema);
_env.AssertValue(parent);
_env.AssertValue(schema.Feature);
_parent = parent;
InputRoleMappedSchema = schema;
var genericMapper = parent.GenericMapper.Bind(_env, schema);
_genericRowMapper = genericMapper as ISchemaBoundRowMapper;
if (parent.Stringify)
{
var builder = new SchemaBuilder();
builder.AddColumn(DefaultColumnNames.FeatureContributions, TextType.Instance, null);
_outputSchema = builder.GetSchema();
if (InputSchema.HasSlotNames(InputRoleMappedSchema.Feature.Index, InputRoleMappedSchema.Feature.Type.VectorSize))
{
InputSchema.GetMetadata(MetadataUtils.Kinds.SlotNames, InputRoleMappedSchema.Feature.Index,
ref _slotNames);
}
else
{
_slotNames = VBufferUtils.CreateEmpty <ReadOnlyMemory <char> >(InputRoleMappedSchema.Feature.Type.VectorSize);
}
}
else
{
_outputSchema = Schema.Create(new FeatureContributionSchema(_env, DefaultColumnNames.FeatureContributions,
new VectorType(NumberType.R4, schema.Feature.Type as VectorType),
InputSchema, InputRoleMappedSchema.Feature.Index));
}
_outputGenericSchema = _genericRowMapper.OutputSchema;
OutputSchema = new CompositeSchema(new Schema[] { _outputGenericSchema, _outputSchema, }).AsSchema;
}
private static async Task PublishEventsAsync(
GridConfiguration gridConfig,
EventGridEdgeClient egClient,
CancellationToken cancellationToken)
{
Console.WriteLine($"Will publish events every {gridConfig.PublishIntervalInSeconds} seconds");
string topicName = gridConfig.Topic.Name;
InputSchema inputSchema = GetTopicInputSchema(gridConfig);
int publishIntervalInSeconds = gridConfig.PublishIntervalInSeconds;
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
using (CancellationTokenSource cts = new CancellationTokenSource(30 * 1000))
{
EventGridEvent evtPayload = (EventGridEvent)CreateEvent(topicName, inputSchema);
await egClient.Events.PublishJsonAsync(topicName : topicName, evtPayload.Id, payload : evtPayload, contentType : ApplicationJsonMTHV, cts.Token).ConfigureAwait(false);
Console.WriteLine($"Published event {JsonConvert.SerializeObject(evtPayload)} to eventgrid module ...");
Console.WriteLine();
}
}
catch (Exception e)
{
Console.WriteLine($"Failed to publish event to topic {topicName}. Reason: {e.ToString()}");
}
Thread.Sleep(publishIntervalInSeconds * 1000);
}
}
public IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetInputColumnRoles()
{
return(InputSchema.GetColumnRoles().Select(kvp => new KeyValuePair <RoleMappedSchema.ColumnRole, string>(kvp.Key, kvp.Value.Name)));
}
/// <summary>
/// For adding to the maintained list of what entities are joint together by this operator
/// </summary>
/// <param name="entityAlias1">the alias on the left side of the join</param>
/// <param name="entityAlias2">the alias on the right side of the join</param>
internal void AddJoinPair(JoinKeyPair joinKeyPair)
{
Debug.Assert(InputSchema.Where(f => f is EntityField && f.FieldAlias == joinKeyPair.NodeAlias).Count() == 1);
Debug.Assert(InputSchema.Where(f => f is EntityField && f.FieldAlias == joinKeyPair.RelationshipOrNodeAlias).Count() == 1);
_joinPairs.Add(joinKeyPair);
}
internal override void PropagateDateTypesForOutSchema()
{
// projection may alter the schema with calculated columns
// we calculate the data type of all the fields in the output schema
// using type evaluation method on the QueryExpression
var exprToOutputMap = ProjectionMap.ToDictionary(
kv => kv.Key, // key is output alias
kv => new { Expr = kv.Value, Field = OutputSchema.First(f => f.FieldAlias == kv.Key) } // value is the corresponding field object and expression
);
foreach (var map in exprToOutputMap)
{
// toggle the fact if any of the output column requires aggregation
HasAggregationField = HasAggregationField ||
(map.Value.Expr.GetChildrenQueryExpressionType <QueryExpressionAggregationFunction>().Count() > 0);
var allPropertyReferences = map.Value.Expr.GetChildrenQueryExpressionType <QueryExpressionProperty>();
if (map.Value.Field is EntityField)
{
// This can only be direct exposure of entity (as opposed to deference of a particular property)
// We just copy of the fields that the entity can potentially be dereferenced
Debug.Assert(allPropertyReferences.Count() == 1);
var varName = allPropertyReferences.First().VariableName;
var matchInputField = InputSchema.First(f => f.FieldAlias == varName);
map.Value.Field.Copy(matchInputField);
}
else
{
// This can be a complex expression involve multiple field/column references
// We will compute the type of the expression
Debug.Assert(map.Value.Field is ValueField);
// first of all, bind the type to the variable references
foreach (var prop in allPropertyReferences)
{
var varName = prop.VariableName;
var propName = prop.PropertyName;
Debug.Assert(prop.VariableName != null);
var matchedField = InputSchema.FirstOrDefault(f => f.FieldAlias == varName);
if (matchedField == null)
{
throw new TranspilerBindingException($"Failed to find input matching field alias {varName}");
}
if (string.IsNullOrEmpty(propName))
{
// direct reference to an alias (value column or entity column)
if (matchedField is ValueField)
{
prop.DataType = (matchedField as ValueField).FieldType;
}
else
{
// entity field reference in a single field expression
// this is valid only in handful situations, such as Count(d), Count(distinct(d))
// in such case, we populate the Entity object with correct entity type so that code generator can use it later
Debug.Assert(matchedField is EntityField);
var matchedEntity = matchedField as EntityField;
prop.Entity = matchedEntity.Type == EntityField.EntityType.Node ?
new NodeEntity()
{
EntityName = matchedEntity.EntityName, Alias = matchedEntity.FieldAlias
} as Entity:
new RelationshipEntity()
{
EntityName = matchedEntity.EntityName, Alias = matchedEntity.FieldAlias
} as Entity;
}
}
else
{
// property dereference of an entity column
if (!(matchedField is EntityField))
{
throw new TranspilerBindingException($"Failed to dereference property {propName} for alias {varName}, which is not an alias of entity type as expected");
}
var entField = matchedField as EntityField;
var entPropField = entField.EncapsulatedFields.FirstOrDefault(f => f.FieldAlias == propName);
if (entPropField == null)
{
throw new TranspilerBindingException($"Failed to dereference property {propName} for alias {varName}, Entity type {entField.BoundEntityName} does not have a property named {propName}");
}
entField.AddReferenceFieldName(propName);
prop.DataType = entPropField.FieldType;
}
}
// do data type evaluation
var evalutedType = map.Value.Expr.EvaluateType();
var outField = map.Value.Field as ValueField;
outField.FieldType = evalutedType;
}
}
}
public override RowMapperColumnInfo[] GetOutputColumns() => _parent.ColumnPairs.Select((x, idx) => new RowMapperColumnInfo(x.output, InputSchema.GetColumnType(ColMapNewToOld[idx]), null)).ToArray();
