private static Tuple <Type, string> Generate <TKey, TSource, TResult>(
string ingressType,
string partitionString,
Expression <Func <TSource, TKey> > partitionExpression,
Expression <Func <TSource, long> > startEdgeExpression,
Expression <Func <TSource, long> > endEdgeExpression,
string latencyOption,
string diagnosticOption,
FuseModule fuseModule)
{
var template = new TemporalIngressTemplate(
string.Format("GeneratedTemporalIngress_{0}", TemporalIngressSequenceNumber++),
typeof(TKey), typeof(TSource), fuseModule?.OutputType ?? typeof(TResult));
var tm = new TypeMapper(template.keyType, template.payloadType, template.resultType);
var gps = tm.GenericTypeVariables(template.keyType, template.payloadType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.payloadOrResult = tm.CSharpNameFor(template.resultRepresentation.RepresentationFor);
var batchGeneratedFrom_TKey_TPayload = string.IsNullOrEmpty(partitionString)
? Transformer.GetBatchClassName(template.keyType, template.resultType)
: Transformer.GetBatchClassName(typeof(PartitionKey <TKey>), template.resultType);
var keyAndPayloadGenericParameters = tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.keyOrNothing = string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ";
template.diagnosticOption = diagnosticOption;
template.needsStreamEvent = ingressType == "StreamEvent";
template.genericArguments = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + template.TPayload;
template.adjustedGenericArgs = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + (!fuseModule.IsEmpty && Config.AllowFloatingReorderPolicy ? template.TResult : template.TPayload);
template.emptyOrPartition = string.IsNullOrEmpty(partitionString) ? "Microsoft.StreamProcessing.Empty.Default" : "new PartitionKey<" + template.TKey + ">(value.PartitionKey)";
template.partitionString = partitionString;
template.baseStructure = partitionString + "StreamEvent<" + template.genericArguments + ">";
template.inheritBase = (ingressType != "StreamEvent") ? template.TPayload : template.baseStructure;
template.ingressType = ingressType;
template.latencyOption = latencyOption;
template.partitionFunction = (x => partitionExpression == null ?
string.Empty : partitionExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ partitionExpression.Parameters.Single(), x }
}));
template.startEdgeFunction = (x => startEdgeExpression == null ?
string.Empty : startEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ startEdgeExpression.Parameters.Single(), x }
}));
template.endEdgeFunction = (x => endEdgeExpression == null ?
"StreamEvent.InfinitySyncTime" :
endEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ endEdgeExpression.Parameters.Single(), x }
}));
template.fusionOption = fuseModule.IsEmpty ? "Simple" :
(Config.AllowFloatingReorderPolicy ? "Disordered" : "Fused");
template.resultMightBeNull = template.resultType.CanContainNull();
if (!fuseModule.IsEmpty)
{
template.valueString = fuseModule.Coalesce <TSource, TResult, TKey>("value.SyncTime", "value.OtherTime", "value.Payload", template.emptyOrPartition, out template.leadingText, out template.trailingText);
template.generatedEndTimeVariable = "generatedEndTimeVariable";
if (Config.AllowFloatingReorderPolicy)
{
template.valueString = "value.Payload";
template.generatedEndTimeVariable = "value.OtherTime";
}
}
return(template.Generate <TKey, TSource, TResult>(typeof(IStreamable <,>)));
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n");
var typeInfos = types.Select(t =>
Tuple.Create(String.Format("_{0}_Pool", Transformer.GetValidIdentifier(t)), t.GetCSharpSourceSyntax()));
this.Write("\r\npublic sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("<_Key, _Payload> : MemoryPool<_Key, _Payload>\r\n{\r\n");
foreach (var t in typeInfos)
{
this.Write("\r\n public ColumnPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item2));
this.Write("> ");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item1));
this.Write(";\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public bool Get(" +
"out ColumnBatch<");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item2));
this.Write("> result)\r\n {\r\n return ");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item1));
this.Write(".Get(out result);\r\n }\r\n");
}
this.Write("\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() : base(true)\r\n {\r\n");
foreach (var t in typeInfos)
{
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item1));
this.Write(" = MemoryManager.GetColumnPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(t.Item2));
this.Write(">();\r\n");
}
this.Write("\r\n }\r\n\r\n}");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" +
"l<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(@"> pool;
private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator;
[DataMember]
private FastMap<ActiveEventLeft> leftEdgeMap = new FastMap<ActiveEventLeft>(DefaultCapacity);
[DataMember]
private FastMap<ActiveEventRight> rightEdgeMap = new FastMap<ActiveEventRight>(DefaultCapacity);
private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TResult));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@" output;
[DataMember]
private long nextLeftTime = long.MinValue;
[DataMember]
private bool isLeftComplete;
[DataMember]
private long nextRightTime = long.MinValue;
[DataMember]
private bool isRightComplete;
");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> qu" +
"eryPlanGenerator)\r\n : base(stream, observer)\r\n {\r\n pool = Memor" +
"yManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
GetOutputBatch();
this.Write(" this.queryPlanGenerator = queryPlanGenerator;\r\n }\r\n\r\n protected ove" +
"rride void FlushContents()\r\n {\r\n if (output.Count == 0) return;\r\n " +
" output.Seal();\r\n this.Observer.OnNext(output);\r\n ");
GetOutputBatch();
this.Write(@" }
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + rightEdgeMap.Count;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void DisposeState() => this.output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> leftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" +
"Free, out bool rightBatchFree)\r\n {\r\n var generatedLeftBatch = leftBatc" +
"h as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(";\r\n var generatedRightBatch = rightBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
leftBatchFree = rightBatchFree = true;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
FastMap<ActiveEventRight>.FindTraverser rightEdges = default;
while (true)
{
bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime;
bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime;
if (nextLeftTime <= nextRightTime)
{
if (leftPunctuation)
{
AddPunctuationToBatch(nextLeftTime);
}
else
{
// ProcessLeftStartEdge
if (true)
{
bool first = true;
");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = default;
var hash = leftBatch.hash.col[leftBatch.iter];
if (rightEdgeMap.Find(hash, ref rightEdges))
{
int rightIndex;
while (rightEdges.Next(out rightIndex))
{
if (first) { key = leftBatch.key.col[leftBatch.iter]; first = false; }
//if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key))
var activeEventRight = rightEdgeMap.Values[rightIndex];
var rightKey = activeEventRight.Key;
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey")));
this.Write(@")
{
//payload = leftBatch[leftBatch.iter];
//OutputStartEdge(nextLeftTime, ref key, ref payload, ref rightEdgeMap.Values[rightIndex].Payload, hash);
");
OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "leftBatch.iter", "activeEventRight", "hash");
this.Write(@" }
}
}
if (!isRightComplete)
{
int newIndex = leftEdgeMap.Insert(hash);
if (first) key = leftBatch.key.col[leftBatch.iter];
leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, leftBatch.iter);
}
}
}
leftBatch.iter++;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
}
else
{
if (rightPunctuation)
{
AddPunctuationToBatch(nextRightTime);
}
else
{
// ProcessRightStartEdge
");
ProcessRightStartEdge2(
"nextRightTime",
"generatedRightBatch",
"rightBatch.iter",
"rightBatch.hash.col[rightBatch.iter]");
this.Write(@" }
rightBatch.iter++;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" +
"dLeftBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextLeftTime(batch.vsync.col[batch.iter]);
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
// ProcessLeftStartEdge
FastMap<ActiveEventRight>.FindTraverser rightEdges = default;
if (true)
{
bool first = true;
");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = default;
var hash = batch.hash.col[batch.iter];
if (rightEdgeMap.Find(hash, ref rightEdges))
{
int rightIndex;
while (rightEdges.Next(out rightIndex))
{
if (first) { key = batch.key.col[batch.iter]; first = false; }
//if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key))
var activeEventRight = rightEdgeMap.Values[rightIndex];
var rightKey = activeEventRight.Key;
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey")));
this.Write(")\r\n {\r\n //payload = batch[batch" +
".iter];\r\n //OutputStartEdge(nextLeftTime, ref key, re" +
"f payload, ref rightEdgeMap.Values[rightIndex].Payload, hash);\r\n " +
" ");
OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "batch.iter", "activeEventRight", "hash");
this.Write(@" }
}
}
if (!isRightComplete)
{
int newIndex = leftEdgeMap.Insert(hash);
if (first) key = batch.key.col[batch.iter];
//payload = batch[batch.iter]; // potential rare recomputation
//leftEdgeMap.Values[newIndex].Populate(ref key, ref payload);
leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, batch.iter);
}
}
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessRightBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" +
"dBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextRightTime(batch.vsync.col[batch.iter]);
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
// ProcessRightStartEdge
");
ProcessRightStartEdge2(
"nextRightTime",
"generatedBatch",
"batch.iter",
"batch.hash.col[batch.iter]");
this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n\r\n [MethodImpl(MethodImplOpti" +
"ons.AggressiveInlining)]\r\n private bool GoToVisibleRow<TPayload>(StreamMessag" +
"e<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", TPayload> batch)\r\n {\r\n while (batch.iter < batch.Count && (batch.bitv" +
"ector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.c" +
"ol[batch.iter] >= 0)\r\n {\r\n batch.iter++;\r\n }\r\n\r\n " +
" return (batch.iter != batch.Count);\r\n }\r\n\r\n [MethodImpl(MethodImplOptions" +
".AggressiveInlining)]\r\n private void UpdateNextLeftTime(long time)\r\n {\r\n " +
" nextLeftTime = time;\r\n if (nextLeftTime == StreamEvent.InfinitySync" +
"Time)\r\n {\r\n isLeftComplete = true;\r\n }\r\n }\r\n\r\n [M" +
"ethodImpl(MethodImplOptions.AggressiveInlining)]\r\n private void UpdateNextRig" +
"htTime(long time)\r\n {\r\n nextRightTime = time;\r\n if (nextRightTi" +
"me == StreamEvent.InfinitySyncTime)\r\n {\r\n isRightComplete = tr" +
"ue;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r" +
"\n private void AddPunctuationToBatch(long start)\r\n {\r\n if (start > " +
"lastCTI)\r\n {\r\n lastCTI = start;\r\n\r\n int index = out" +
"put.Count++;\r\n output.vsync.col[index] = start;\r\n output.v" +
"other.col[index] = StreamEvent.PunctuationOtherTime;\r\n output.key.col" +
"[index] = default;\r\n output[index] = default;\r\n output.has" +
"h.col[index] = 0;\r\n output.bitvector.col[index >> 6] |= (1L << (index" +
" & 0x3f));\r\n\r\n if (output.Count == Config.DataBatchSize) FlushContent" +
"s();\r\n }\r\n }\r\n\r\n [DataContract]\r\n private struct ActiveEventLeft" +
"\r\n {\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n ");
foreach (var f in this.leftFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public vo" +
"id Populate(");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " +
" ");
foreach (var f in this.leftFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n\r\n [DataContract]\r\n private" +
" struct ActiveEventRight\r\n {\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n ");
foreach (var f in this.rightFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TRight));
this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " +
" ");
foreach (var f in this.rightFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}\r\n\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
if (this.resultType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TPayload: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n// Source Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName))));
this.Write("\r\n// Destination Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.resultFields.Select(f => f.OriginalName))));
this.Write("\r\n// Computed Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName))));
this.Write("\r\n// ProjectionReturningResultInstance: ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.projectionReturningResultInstance.ExpressionToCSharp()));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" +
"rator;\r\n\r\n [DataMember]\r\n private int iter;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" batch;\r\n\r\n // Fields used to point directly to the arrays within the result b" +
"atch\r\n private long[] dest_vsync;\r\n private long[] dest_vother;\r\n priva" +
"te ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("[] destkey;\r\n private int[] dest_hash;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" +
"ool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(@";
this.queryPlanGenerator = queryPlanGenerator;
MyAllocate();
}
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
private void MyAllocate()
{
pool.Get(out genericBatch);
genericBatch.Allocate();
this.batch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@";
this.UpdatePointers();
}
protected override void UpdatePointers()
{
// Assign pointers to bookkeeping arrays
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
protected override void DisposeState() => this.batch.Free();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> _inBatch)\r\n {\r\n var batch = _inBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters));
this.Write(";\r\n\r\n var count = batch.Count;\r\n this.batch.iter = batch.iter;\r\n\r\n " +
" // Create locals that point directly to the arrays within the columns in " +
"the destination batch.\r\n");
foreach (var f in this.computedFields.Keys)
{
if (f.OptimizeString())
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the source batch.\r\n");
foreach (var f in this.fields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n var srckey = batch.key.col;\r\n");
if (this.hasKey)
{
this.Write(" var key_col = srckey; // hack until MakeColumnOriented is fixed\r\n");
}
this.Write(@"
fixed (long* src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
fixed (int* src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
");
if (this.StartEdgeParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName));
this.Write(" = src_vsync[i] < src_vother[i] ? src_vsync[i] : src_vother[i];\r\n");
}
if (this.useEnumerator)
{
this.Write(" var enumerator = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(".GetEnumerator();\r\n while (enumerator.MoveNext())\r\n");
}
else
{
if (!this.enumerableRepeatSelector)
{
this.Write("\r\n var e_prime = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(";\r\n");
}
this.Write("\r\n");
if (this.keyParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyParameterName));
this.Write(" = srckey[i];\r\n");
}
this.Write(" for (int _x = 0; _x < ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.loopCounter));
this.Write("; _x++)\r\n");
}
this.Write("\r\n {\r\n dest_vsync[iter] = src_vsync[i];" +
"\r\n dest_vother[iter] = src_vother[i];\r\n");
if (this.useEnumerator)
{
this.Write(" this.batch[iter] = enumerator.Current;\r\n");
}
else if (this.enumerableRepeatSelector)
{
if (this.projectionReturningResultInstance != null)
{
this.Write(" this.batch[iter] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.projectionReturningResultInstance.ExpressionToCSharp()));
this.Write(";\r\n");
}
else
{
foreach (var kv in this.computedFields)
{
var f = kv.Key;
var v = kv.Value.ExpressionToCSharp();
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[iter] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
}
}
}
else
{
this.Write(" this.batch[iter] = e_prime;\r\n");
}
this.Write(@"
destkey[iter] = srckey[i];
dest_hash[iter] = src_hash[i];
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n this.batch.iter = batch.iter;\r\n " +
" }\r\n }\r\n");
if (this.useEnumerator)
{
this.Write("\r\n enumerator.Dispose();\r\n");
}
this.Write(@" }
else if (src_vother[i] < 0)
{
dest_vsync[iter] = src_vsync[i];
dest_vother[iter] = src_vother[i];
destkey[iter] = default;
this.batch[iter] = default;
dest_hash[iter] = src_hash[i];
this.batch.bitvector.col[(iter) >> 6] |= (1L << ((iter) & 0x3f));
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(" this.batch.iter = batch.iter;\r\n }\r\n " +
" }\r\n }\r\n\r\n } // end src_hash, src_bv, src_vsync, s" +
"rc_vother\r\n\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write(@"
batch.Free();
}
protected override void FlushContents()
{
if (iter == 0) return;
this.batch.Count = iter;
this.batch.Seal();
this.Observer.OnNext(this.batch);
iter = 0;
MyAllocate();
}
public override int CurrentlyBufferedOutputCount => iter;
public override int CurrentlyBufferedInputCount => 0;
}
");
return(this.GenerationEnvironment.ToString());
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string errorMessages = null;
try
{
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++);
var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.destinationFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
// Don't create a parameter substitution for the start edge parameter. That will just remain in the
// body of the result selector and will be set as a local variable in the generated code.
if (stream.HasKey)
{
var keyRepresentation = new ColumnarRepresentation(keyType, "key");
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(keyParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation,
}));
}
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(payloadParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
}));
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge);
if (projectionResult.Error)
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't transform a selector!");
}
return(Tuple.Create((Type)null, errorMessages));
}
template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null;
template.computedFields = projectionResult.ComputedFields;
template.swingingFields = projectionResult.SwingingFields;
template.unassignedFields = projectionResult.UnmentionedFields;
template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.multiStringOperations = projectionResult.MultiStringOperations;
template.needSourceInstance = projectionResult.NeedsSourceInstance;
var d = new Dictionary <MyFieldInfo, int>();
foreach (var f in template.swingingFields)
{
var target = f.Item1;
var source = f.Item2;
if (!d.ContainsKey(source))
{
d.Add(source, 0);
}
d[source] = d[source] + 1;
}
template.swungFieldsCount = d;
template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf)));
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(
typeof(TKey), typeof(TPayload), typeof(TResult), typeof(IStreamable <,>));
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
[assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")]
// OuterKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write("\r\n// InnerKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write("\r\n// InnerResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n// ProjectionReturningResultInstance: ");
this.Write(this.ToStringHelper.ToStringWithCulture(ProjectionReturningResultInstance.ExpressionToCSharp()));
this.Write("\r\n\r\n");
var memoryPoolGenericParameters = string.Format("<{0}, {1}>", TOuterKey, TResult);
if (resultType.MemoryPoolHasGetMethodFor())
{
memoryPoolGenericParameters = string.Empty;
}
var memoryPoolClassName = Transformer.GetMemoryPoolClassName(this.outerKeyType, this.resultType) + memoryPoolGenericParameters;
var inputKey = ungroupingFromCompound ? "CompoundGroupKey<" + TOuterKey + ", " + TInnerKey + ">" : TInnerKey;
this.Write("\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" :\r\n Pipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">, IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(inputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write(">\r\n{\r\n private readonly ");
this.Write(this.ToStringHelper.ToStringWithCulture(memoryPoolClassName));
this.Write(" outPool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanG" +
"enerator;\r\n\r\n");
if (ungroupingToUnit)
{
this.Write(" private readonly ColumnBatch<Microsoft.StreamProcessing.Empty> unitColumn;\r\n " +
" private readonly ColumnBatch<int> unitHashColumn;\r\n");
}
else
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", int> outerHashCode;\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" private readonly ColumnBatch<");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("> sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n this.outPool = MemoryManager.Ge" +
"tMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true) as ");
this.Write(this.ToStringHelper.ToStringWithCulture(memoryPoolClassName));
this.Write(";\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n\r\n");
if (ungroupingToUnit)
{
this.Write(" this.outPool.GetKey(out this.unitColumn);\r\n this.outPool.Get(out t" +
"his.unitHashColumn);\r\n Array.Clear(this.unitHashColumn.col, 0, this.unitH" +
"ashColumn.col.Length);\r\n");
}
else
{
this.Write(" this.outerHashCode = stream.Properties.KeyEqualityComparer.GetGetHashCode" +
"Expr().Compile();\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" this.outPool.Get(out sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n Array.Clear(sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col, 0, sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col.Length);\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
protected override void DisposeState()
{
");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.Return();\r\n this.unitHashColumn.Return();\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
this.Write(" }\r\n\r\n public unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(inputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write("> batch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchGenericParameters));
this.Write(" inputBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchGenericParameters));
this.Write(";\r\n\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> tmp; // Need this type to call Get with so the right subtype will be returned\r\n" +
" outPool.Get(out tmp);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(" resultBatch = tmp as ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(";\r\n\r\n");
foreach (var f in (this.ProjectionReturningResultInstance != null ? new ColumnarRepresentation(this.resultType).AllFields : this.computedFields.Keys))
{
this.Write("\r\n outPool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit)
{
this.Write("\r\n tmp.hash = batch.hash.MakeWritable(outPool.intPool);\r\n");
}
this.Write(" var count = batch.Count;\r\n\r\n tmp.vsync = batch.vsync;\r\n tmp" +
".vother = batch.vother;\r\n tmp.bitvector = batch.bitvector;\r\n\r\n // " +
"Assign the swinging fields\r\n");
foreach (var tuple in this.swingingFields)
{
var destField = tuple.Item1.Name;
var sourceField = tuple.Item2.Name;
this.Write("\r\n resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(";\r\n");
}
this.Write("\r\n");
if (resultType.CanContainNull())
{
this.Write("\r\n outPool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write("\r\n fixed (long *srcbv = batch.bitvector.col) {\r\n");
if (!ungroupingToUnit)
{
this.Write(" fixed (int *desthash = tmp.hash.col) {\r\n");
}
this.Write("\r\n // Get pointers to the arrays for the inner result fields\r\n");
foreach (var f in this.innerResultRepresentation.AllFields)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
this.Write("\r\n // Get pointers to the arrays for the result fields\r\n");
foreach (var f in this.computedFields.Keys)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col) {\r\n");
}
else
{
if (!f.OptimizeString())
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
}
}
this.Write("\r\n");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.IncrementRefCount(1);\r\n tmp.key = unitColumn;\r\n " +
" this.unitHashColumn.IncrementRefCount(1);\r\n tmp.hash = unitHashColum" +
"n;\r\n");
}
this.Write("\r\n");
foreach (var f in this.unassignedFields)
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".IncrementRefCount(1);\r\n resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit || this.computedFields.Any() || this.ProjectionReturningResultInstance != null)
{
if (!ungroupingToUnit)
{
this.Write(" var srckey = batch.key.col;\r\n var destkey = tmp.key.col;\r\n" +
"");
}
this.Write("\r\n for (int i = 0; i < count; i++)\r\n {\r\n if " +
"((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n " +
" // Need to add empty strings to keep multistring indexing consistent\r\n");
foreach (var kv in this.computedFields)
{
var field = kv.Key;
if (field.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(field.Name));
this.Write(".AddString(string.Empty);\r\n");
}
}
this.Write(" continue;\r\n }\r\n");
if (!ungroupingToUnit)
{
this.Write(" destkey[i] = srckey[i].outerGroup;\r\n desthash[i] =" +
" this.outerHashCode(destkey[i]);\r\n");
}
if (this.ProjectionReturningResultInstance != null)
{
var map = new Dictionary <System.Linq.Expressions.ParameterExpression, string>();
map.Add(this.keyParameter, "batch.key.col[i].InnerGroup");
var v = this.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(map);
this.Write(" resultBatch[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
else
{
foreach (var kv in this.computedFields)
{
var f = kv.Key;
string v;
var map = new Dictionary <System.Linq.Expressions.ParameterExpression, string>();
map.Add(this.keyParameter, "batch.key.col[i].InnerGroup");
v = kv.Value.ExpressionToCSharpStringWithParameterSubstitution(map);
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
} // end foreach
}
this.Write(" }\r\n");
}
this.Write("\r\n");
foreach (var f in this.computedFields.Keys.Where(fld => fld.canBeFixed))
{
this.Write("\r\n } // end of fixed for ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit)
{
this.Write("\r\n } // end of fixed for desthash\r\n");
}
this.Write(" } // end of fixed for srcbv\r\n\r\n tmp.Count = count;\r\n\r\n batc" +
"h.ReleaseKey();\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n batch.hash.Return();\r\n");
}
this.Write("\r\n #region Return source columns as necessary.\r\n // This is all fie" +
"lds from the input batch *except* for any swinging fields.\r\n");
foreach (var f in this.innerResultRepresentation.AllFields.Where(f => !this.swingingFields.Any(tup => tup.Item2.Name == f.Name)))
{
if (f.OptimizeString())
{
this.Write(" inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Dispose();\r\n");
}
else
{
this.Write(" inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
}
this.Write(" #endregion\r\n\r\n");
if (innerResultType.CanContainNull())
{
this.Write("\r\n inputBatch._nullnessvector.ReturnClear();\r\n");
}
this.Write("\r\n batch.Return();\r\n\r\n tmp.Seal();\r\n this.Observer.OnNext(tm" +
"p);\r\n }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
[assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")]
");
List <string> genericParamList = new List <string>();
var TOuterKey = outerKeyType.GetCSharpSourceSyntax(ref genericParamList);
var outerKeyGenericParameterCount = genericParamList.Count;
var TSource = sourceType.GetCSharpSourceSyntax(ref genericParamList);
var sourceGenericParameterCount = genericParamList.Count;
var sourceGenericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty;
var TInnerKey = innerKeyType.GetCSharpSourceSyntax(ref genericParamList);
var genericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty;
var outKeyInnerKeyGenericParameterList = genericParamList.Take(outerKeyGenericParameterCount).Concat(genericParamList.Skip(sourceGenericParameterCount));
var TOuterKeyTInnerKeyGenericParameters = 0 < outKeyInnerKeyGenericParameterList.Count() ? "<" + String.Join(",", outKeyInnerKeyGenericParameterList) + ">" : string.Empty;
var resultBatchGenericParameters = genericParameters;
var resultBatchClassType = isFirstLevelGroup
? Transformer.GetBatchClassName(innerKeyType, sourceType)
: Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), sourceType);
var sourceBatchClassType = Transformer.GetBatchClassName(outerKeyType, sourceType);
var innerKeyIsAnonymous = innerKeyType.IsAnonymousTypeName();
if (innerKeyIsAnonymous)
{
transformedKeySelectorAsString = string.Format("({0})Activator.CreateInstance(typeof({0}), {1} )", TInnerKey, transformedKeySelectorAsString);
}
var outputKey = !isFirstLevelGroup ? "CompoundGroupKey<" + TOuterKey + ", " + TInnerKey + ">" : TInnerKey;
var nestedInfix = !isFirstLevelGroup ? "Nested" : string.Empty;
var ungroupingToUnit = isFirstLevelGroup && innerKeyType == typeof(Empty);
var swingingKeyFromField = isFirstLevelGroup && !this.swingingField.Equals(default(MyFieldInfo));
this.Write("\r\n// TOuterKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write("\r\n// TInnerKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write("\r\n// TSource: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" :\r\n Pipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">, IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">\r\n{\r\n");
if (innerKeyIsAnonymous)
{
this.Write(" [SchemaSerialization]\r\n private readonly Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int>> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int> innerHashCode;\r\n");
}
this.Write(" private readonly MemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> l1Pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanG" +
"enerator;\r\n\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n private readonly ColumnBatch<Microsoft.StreamProcessing.Empty> unitColumn;\r" +
"\n private readonly ColumnBatch<int> unitHashColumn;\r\n");
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> observer,\r\n Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int>> keyComparer,\r\n Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(">> keySelector,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerato" +
"r)\r\n : base(stream, observer)\r\n {\r\n");
if (innerKeyIsAnonymous)
{
if (isFirstLevelGroup)
{
this.Write(" var _keySelector = keySelector.Compile();\r\n");
}
this.Write(" this.keyComparer = keyComparer;\r\n this.innerHashCode = keyComparer" +
".Compile();\r\n");
}
this.Write(" this.l1Pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">(true);\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n this.l1Pool.GetKey(out this.unitColumn);\r\n this.l1Pool.Get(out t" +
"his.unitHashColumn);\r\n Array.Clear(this.unitHashColumn.col, 0, this.unitH" +
"ashColumn.col.Length);\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
protected override void FlushContents() { }
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
protected override void DisposeState()
{
");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.Return();\r\n this.unitHashColumn.Return();\r\n");
}
this.Write(" }\r\n\r\n public unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> batch)\r\n {\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> resultBatchUncast; // Need this type to call Get with so the right subtype will" +
" be returned\r\n this.l1Pool.Get(out resultBatchUncast);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(" resultBatch = resultBatchUncast as ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters));
this.Write(" inputBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters));
this.Write(";\r\n\r\n resultBatch.vsync = batch.vsync;\r\n resultBatch.vother = batch" +
".vother;\r\n");
foreach (var f in this.fields)
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
if (this.payloadMightBeNull)
{
this.Write(" resultBatch._nullnessvector = inputBatch._nullnessvector;\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit && !swingingHashColumn)
{
this.Write("\r\n resultBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool);\r\n");
}
this.Write(" resultBatch.bitvector = batch.bitvector;\r\n\r\n");
if (!ungroupingToUnit && (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo))))
{
this.Write(" this.l1Pool.GetKey(out resultBatchUncast.key);\r\n");
}
this.Write("\r\n resultBatch.Count = batch.Count;\r\n\r\n var count = batch.Count;\r\n\r" +
"\n");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.IncrementRefCount(1);\r\n resultBatch.key = unitColu" +
"mn;\r\n this.unitHashColumn.IncrementRefCount(1);\r\n resultBatch.hash" +
" = unitHashColumn;\r\n");
}
else
{
this.Write("\r\n var src_key = batch.key.col;\r\n");
if (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo)))
{
this.Write("\r\n var dest_key = resultBatch.key.col;\r\n");
}
this.Write(" var dest_payload = resultBatch;\r\n\r\n");
if (swingingKeyFromField)
{
this.Write(" // The key selector was just e => e.f *and* isFirstLevelGroup so just swi" +
"ng the key field\r\n");
if (this.swingingField.OptimizeString())
{
this.Write("\r\n resultBatch.key = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name));
this.Write(".ToColumnBatch(this.l1Pool.stringPool, resultBatch.bitvector);\r\n");
}
else
{
this.Write("\r\n resultBatch.key = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name));
this.Write(";\r\n resultBatch.key.IncrementRefCount(1);\r\n");
}
}
this.Write("\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n");
if (f.OptimizeString())
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(vectorHashCodeInitialization));
this.Write("\r\n\r\n");
if (!swingingKeyFromField || !swingingHashColumn)
{
this.Write("\r\n fixed (long* src_bv = batch.bitvector.col) {\r\n");
if (!ungroupingToUnit)
{
this.Write(" fixed (int* dest_hash = resultBatch.hash.col) {\r\n");
}
this.Write(" for (int i = 0; i < count; i++)\r\n {\r\n if ((" +
"src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) continue;\r\n\r\n");
if (!swingingKeyFromField || !this.swingingField.OptimizeString())
{
this.Write("\r\n var key = ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.transformedKeySelectorAsString));
this.Write(";\r\n");
}
if (innerKeyIsAnonymous)
{
this.Write(" var innerHash = this.innerHashCode(key);\r\n");
}
else if (swingingHashColumn)
{
this.Write("\r\n // don\'t compute hash because that was done by calling MultiStr" +
"ing.GetHashCode\r\n");
}
else
{
this.Write(" var innerHash = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedHashCodeComputation));
this.Write(";\r\n");
}
this.Write("\r\n");
if (isFirstLevelGroup)
{
if (!this.swingingField.Equals(default(MyFieldInfo)))
{
this.Write(" // no assignment to key, pointer was swung!\r\n");
}
else
{
this.Write(" dest_key[i] = key;\r\n");
}
if (!swingingHashColumn)
{
this.Write("\r\n dest_hash[i] = innerHash;\r\n");
}
}
else
{
this.Write(" var hash = dest_hash[i] ^ innerHash;\r\n dest_key[i]" +
".outerGroup = src_key[i];\r\n dest_key[i].innerGroup = key;\r\n " +
" dest_key[i].hashCode = hash;\r\n dest_hash[i] = hash;\r\n");
}
this.Write(" }\r\n");
if (!ungroupingToUnit)
{
this.Write(" } // end of fixed for dest_hash\r\n");
}
this.Write(" } // end of fixed for src_bv\r\n");
}
this.Write("\r\n ");
if (!String.IsNullOrWhiteSpace(vectorHashCodeInitialization) && !swingingHashColumn)
{
this.Write("\r\n this.hashCodeVector.Return();\r\n ");
}
this.Write("\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n");
}
this.Write("\r\n");
}
this.Write(" batch.ReleaseKey();\r\n");
if (ungroupingToUnit || swingingHashColumn)
{
this.Write("\r\n batch.hash.Return();\r\n");
}
this.Write("\r\n batch.Return();\r\n\r\n this.Observer.OnNext(resultBatch);\r\n }\r\n}" +
"\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n// *********************************************************************\r\n// Co" +
"pyright (c) Microsoft Corporation. All rights reserved.\r\n// Licensed under the " +
"MIT License\r\n// ****************************************************************" +
"*****\r\n");
var resultMessageMemoryPoolGenericParameters = string.Format("<{0}, {1}>", TKey, TResult);
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
resultMessageMemoryPoolGenericParameters = string.Empty;
}
getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(typeof(Empty), resultType),
UnitTResultGenericParameters);
this.Write("\r\n// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n/// <summary>\r\n/// Operator has has no support for ECQ\r\n/// </summary>\r\n[Data" +
"Contract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<Microsoft.StreamProcessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly MemoryPool<Microsoft.StreamProcessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> pool;\r\n\r\n ");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n ");
}
this.Write("\r\n ");
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n ");
}
this.Write("\r\n ");
if (this.useCompiledDeaccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> deaccumulate;\r\n ");
}
this.Write("\r\n ");
if (this.useCompiledDifference)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> difference;\r\n ");
}
this.Write("\r\n ");
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n ");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keySelector;\r\n\r\n ");
if (!this.isUngrouped)
{
this.Write(" [DataMember]\r\n private FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> heldAggregates;\r\n ");
}
else
{
this.Write("\r\n private HeldState currentState;\r\n [DataMember]\r\n private bool isDirty" +
";\r\n ");
}
this.Write("\r\n private StreamMessage<Microsoft.StreamProcessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(typeof(Empty), resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(UnitTResultGenericParameters));
this.Write(" batch;\r\n\r\n [DataMember]\r\n private long lastSyncTime = long.MinValue;\r\n\r\n " +
" private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentKey;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<Microsoft.StreamProcessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<Microsoft.StreamProcessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r" +
"\n : base(stream, observer)\r\n {\r\n this.queryPlanGenerator = quer" +
"yPlanGenerator;\r\n\r\n ");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n ");
}
this.Write(" ");
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n ");
}
this.Write(" ");
if (this.useCompiledDeaccumulate)
{
this.Write(" deaccumulate = aggregate.Deaccumulate().Compile();\r\n ");
}
this.Write(" ");
if (this.useCompiledDifference)
{
this.Write(" difference = aggregate.Difference().Compile();\r\n ");
}
this.Write(" ");
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n ");
}
this.Write("\r\n this.keyComparer = EqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(">.Default;\r\n\r\n this.pool = MemoryManager.GetMemoryPool<Microsoft.StreamPro" +
"cessing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(); // as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(typeof(Empty), this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(resultMessageMemoryPoolGenericParameters));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n ");
if (!this.isUngrouped)
{
this.Write(" var generator = keyComparer.CreateFastDictionary3Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>(1, keyComparer.GetEqualsExpr().Compile(), keyComparer.GetGetHashCode" +
"Expr().Compile(), stream.Properties.QueryContainer);\r\n heldAggregates = g" +
"enerator.Invoke();\r\n ");
}
else
{
this.Write(" isDirty = false;\r\n ");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n\r\n protected override void DisposeSta" +
"te() => this.batch.Free();\r\n\r\n public override int CurrentlyBufferedOutputCou" +
"nt => this.batch.Count;\r\n\r\n public override int CurrentlyBufferedInputCount =" +
"> ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "0" : "heldAggregates.Count"));
this.Write(";\r\n\r\n public override unsafe void OnNext(StreamMessage<Microsoft.StreamProcess" +
"ing.Empty, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
if (!this.isUngrouped)
{
this.Write(" HeldState currentState = null;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentKey = default;\r\n int currentHash = 0;\r\n ");
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_Unit_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(UnitTInputGenericParameters));
this.Write(" batch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_Unit_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(UnitTInputGenericParameters));
this.Write(";\r\n\r\n\r\n ");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(" temporaryOutput;\r\n ");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n\r\n");
}
}
this.Write(@"
fixed (long* col_vsync = batch.vsync.col)
fixed (long* col_vother = batch.vother.col)
fixed (long* col_bv = batch.bitvector.col)
{
for (int i = 0; i < batch.Count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == long.MinValue)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = long.MinValue;
this.batch.key.col[c] = default;
this.batch[c] = default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f));
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
continue;
}
var col_key_i = ");
this.Write(this.ToStringHelper.ToStringWithCulture(keySelector));
this.Write(";\r\n var col_hash_i = ");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparerGetHashCode("col_key_i")));
this.Write(";\r\n\r\n var syncTime = col_vsync[i];\r\n\r\n // Handle ti" +
"me moving forward\r\n if (syncTime > this.lastSyncTime)\r\n " +
" {\r\n ");
if (this.isUngrouped)
{
this.Write(@" if ((currentState != null) && isDirty) // there exists earlier state
{
int _c = this.batch.Count;
this.batch.vsync.col[_c] = currentState.timestamp;
this.batch.vother.col[_c] = StreamEvent.InfinitySyncTime;
{
var currentStateStateState = currentState.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("currentKey", computeResult("currentStateStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n" +
" }\r\n isDirty = false;\r\n " +
" ");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int _c = this.batch.Count;
this.batch.vsync.col[_c] = iter1entry.value.timestamp;
this.batch.vother.col[_c] = StreamEvent.InfinitySyncTime;
{
var iter1entryValueStateState = iter1entry.value.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("iter1entry.key", computeResult("iter1entryValueStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
this.batch.Allocate();
}
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clean();
// reset currentState so that we can later force a re-get and set it dirty
currentState = null;
");
}
this.Write("\r\n // Since sync time changed, set lastSyncTime\r\n " +
" this.lastSyncTime = syncTime;\r\n }\r\n\r\n ");
if (this.isUngrouped)
{
this.Write(" if (currentState == null)\r\n {\r\n " +
" currentState = new HeldState();\r\n currentState.state.state =" +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.state.active = 1; // start edge only
currentState.timestamp = syncTime;
isDirty = true;
}
else
{
if (!isDirty)
{
// Output end edge
int _c = this.batch.Count;
this.batch.vsync.col[_c] = syncTime;
this.batch.vother.col[_c] = currentState.timestamp;
{
var currentStateStateState = currentState.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("currentKey", computeResult("currentStateStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n" +
" currentState.timestamp = syncTime;\r\n " +
" isDirty = true;\r\n }\r\n }\r\n ");
}
else
{
this.Write("\r\n if (currentState == null || ((!");
this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "true" : "false"));
this.Write(") && (currentHash != col_hash_i || !(");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparerEquals("currentKey", "col_key_i")));
this.Write(@"))))
{
// Need to retrieve the key from the dictionary
currentKey = col_key_i;
currentHash = col_hash_i;
int index;
bool heldAggregatesLookup = false;
{
int num = currentHash & 0x7fffffff;
index = num % this.heldAggregates.Size;
do
{
if ((this.heldAggregates.bitvector[index >> 3] & (0x1 << (index & 0x7))) == 0)
{
heldAggregatesLookup = false;
break;
}
if ((currentHash == this.heldAggregates.entries[index].hash) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparerEquals("currentKey", "this.heldAggregates.entries[index].key")));
this.Write(@"))
{
heldAggregatesLookup = true;
break;
}
index++;
if (index == this.heldAggregates.Size)
index = 0;
} while (true);
}
if (!heldAggregatesLookup)
{
// New group. Create new state
currentState = new HeldState();
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.state.active = 1; // start edge only
currentState.timestamp = syncTime;
// No output because initial state is empty
this.heldAggregates.Insert(ref index, currentKey, currentState, currentHash);
}
else
{
// read new currentState from _heldAgg index
currentState = this.heldAggregates.entries[index].value;
if (this.heldAggregates.IsClean(ref index))
{
// Output end edge
int _c = this.batch.Count;
this.batch.vsync.col[_c] = syncTime;
this.batch.vother.col[_c] = currentState.timestamp;
{
var currentStateStateState = currentState.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("currentKey", computeResult("currentStateStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = currentHash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
this.batch.Allocate();
}
currentState.timestamp = syncTime;
this.heldAggregates.SetDirty(ref index);
}
}
}
");
}
this.Write("\r\n currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(";\r\n }\r\n }\r\n\r\n ");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n ");
}
this.Write(" ");
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n ");
}
this.Write("\r\n batch.Release();\r\n batch.Return();\r\n }\r\n\r\n public void OnP" +
"unctuation(long syncTime)\r\n {\r\n\r\n ");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(" temporaryOutput;\r\n ");
foreach (var f in this.outputFields)
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n // Handle time moving forward\r\n if (syncTime > this.lastSyncTime" +
")\r\n {\r\n ");
if (this.isUngrouped)
{
this.Write(@" if ((currentState != null) && isDirty) // need to send start edge if state is dirty
{
int _c = this.batch.Count;
this.batch.vsync.col[_c] = currentState.timestamp;
this.batch.vother.col[_c] = StreamEvent.InfinitySyncTime;
{
var currentStateStateState = currentState.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("currentKey", computeResult("currentStateStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n }\r\n " +
" isDirty = false;\r\n ");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int _c = this.batch.Count;
this.batch.vsync.col[_c] = iter1entry.value.timestamp;
this.batch.vother.col[_c] = StreamEvent.InfinitySyncTime;
{
var iter1entryValueStateState = iter1entry.value.state.state;
");
this.Write(this.ToStringHelper.ToStringWithCulture(finalResultSelector("iter1entry.key", computeResult("iter1entryValueStateState"))));
this.Write(@"
}
this.batch.hash.col[_c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n }\r\n " +
" // Time has moved forward, clean the held aggregates\r\n thi" +
"s.heldAggregates.Clean();\r\n ");
}
this.Write("\r\n // Since sync time changed, set lastSyncTime\r\n this.last" +
"SyncTime = syncTime;\r\n }\r\n if (this.batch.Count > 0)\r\n {\r\n " +
" this.batch.Seal();\r\n this.Observer.OnNext(this.batch);\r\n " +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n }\r\n}");
return(this.GenerationEnvironment.ToString());
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TSource, TInnerKey> > keySelector,
string inlinedHashCodeComputation,
bool nested,
bool powerOf2)
{
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++);
var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var innerKeyRepresentation = keySelector == null ? new ColumnarRepresentation(typeOfTOuterKey) : new ColumnarRepresentation(typeOfTInnerKey);
var template = new ShuffleTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
inlinedHashCodeComputation,
nested, powerOf2)
{
fields = inputMessageRepresentation.AllFields
};
var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName();
if (keySelector != null)
{
var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var keySelectorAsNewExpression = keySelector.Body as NewExpression;
if (innerKeyIsAnonymous && keySelectorAsNewExpression != null)
{
var newPrime = (NewExpression)transformedKeySelector.Body;
template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})",
template.TInnerKey,
string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp())));
}
else
{
template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp();
if (Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name;
template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName);
}
}
}
else
{
template.transformedKeySelectorAsString = string.Empty;
}
template.innerKeyIsAnonymous = innerKeyIsAnonymous;
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("// *********************************************************************\r\n// Copy" +
"right (c) Microsoft Corporation. All rights reserved.\r\n// Licensed under the MI" +
"T License\r\n// ******************************************************************" +
"***\r\n");
var leftBatchTypeName = string.Format("{0}{1}",
Transformer.GetBatchClassName(keyType, leftType),
!String.IsNullOrWhiteSpace(TKeyTLeftGenericParameters) ? "<" + TKeyTLeftGenericParameters + ">" : string.Empty);
var rightBatchTypeName = string.Format("{0}{1}",
Transformer.GetBatchClassName(keyType, rightType),
!String.IsNullOrWhiteSpace(TKeyTRightGenericParameters) ? "<" + TKeyTRightGenericParameters + ">" : string.Empty);
this.Write(@"
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" +
"l<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> pool;\r\n private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPla" +
"nNode> queryPlanGenerator;\r\n\r\n /* Comparer for ordering of join key - should " +
"eventually be a stream property */\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" output;\r\n\r\n [DataMember]\r\n private long nextLeftTime = long.MinValue;\r\n " +
" [DataMember]\r\n private long nextRightTime = long.MinValue;\r\n [DataMember]" +
"\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" nextLeftKey;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" nextRightKey;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentRightKey;\r\n [DataMember]\r\n private List<ActiveEventRight> currentRi" +
"ghtList;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentLeftKey;\r\n [DataMember]\r\n private List<ActiveEventLeft> currentLeft" +
"List;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(@"> observer,
Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator)
: base(stream, observer)
{
currentLeftList = new List<ActiveEventLeft>();
currentRightList = new List<ActiveEventRight>();
pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
this.queryPlanGenerator = queryPlanGenerator;
}
protected override void FlushContents()
{
if (output.Count == 0) return;
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + currentLeftList.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + currentRightList.Count;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void DisposeState() => output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> leftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" +
"Free, out bool rightBatchFree)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" generatedLeftBatch = null;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" generatedRightBatch = null;\r\n\r\n leftBatchFree = rightBatchFree = true;\r\n " +
" generatedLeftBatch = (");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@") leftBatch;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
nextLeftKey = leftBatch.key.col[leftBatch.iter];
generatedRightBatch = (");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(@") rightBatch;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
nextRightKey = rightBatch.key.col[rightBatch.iter];
while (true)
{
bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime;
bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime;
int compare = (leftPunctuation || rightPunctuation) ? 0 : ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(@";
if (compare == 0)
{
if (nextLeftTime <= nextRightTime)
{
// process left
if (leftPunctuation)
{
AddPunctuationToBatch(nextLeftTime);
}
else
{
#region ProcessLeftStartEdge
/*
ProcessLeftStartEdge(
nextLeftTime,
ref leftBatch.key.col[leftBatch.iter],
leftBatch[leftBatch.iter],
leftBatch.hash.col[leftBatch.iter], compare);
*/
{
if (currentRightList.Count > 0)
{
int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentRightList.Count; i++)
{
var t = currentRightList[i];
OutputStartEdge(nextLeftTime > t.Timestamp ? nextLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t, leftBatch.hash.col[leftBatch.iter]);
}
}
else
{
// clear the right array
currentRightList.Clear();
}
}
if (compare >= 0)
{
// update the left array
if ((currentLeftList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey")));
this.Write(" != 0))\r\n {\r\n C" +
"ontract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey")));
this.Write(" > 0);\r\n currentLeftList.Clear();\r\n " +
" }\r\n currentLeftKey = nextLef" +
"tKey;\r\n var leftAE = new ActiveEventLeft();\r\n " +
" leftAE.Populate(ref nextLeftTime, generatedLeftBatch" +
", leftBatch.iter);\r\n currentLeftList.Add(leftAE);" +
"\r\n }\r\n }\r\n " +
" #endregion\r\n }\r\n\r\n leftBatch.iter++;\r\n" +
"\r\n if (!GoToVisibleRow(leftBatch))\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchD" +
"one = false;\r\n return;\r\n }\r\n\r\n " +
" nextLeftTime = leftBatch.vsync.col[leftBatch.iter];\r\n " +
" nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n }\r\n " +
" else\r\n {\r\n // process right\r\n " +
" if (rightPunctuation)\r\n {\r\n " +
" AddPunctuationToBatch(nextRightTime);\r\n }\r\n " +
" else\r\n {\r\n #region ProcessRightSt" +
"artEdge\r\n /* Inlined version of:\r\n " +
" ProcessRightStartEdge(\r\n nextRightTime,\r\n " +
" ref rightBatch.key.col[rightBatch.iter],\r\n " +
" rightBatch[rightBatch.iter],\r\n rightBatch." +
"hash.col[rightBatch.iter], compare);\r\n */\r\n\r\n " +
" if (currentLeftList.Count > 0)\r\n {\r\n " +
" int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
#region OutputStartEdge
/* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp,
ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */
int index = output.Count++;
output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = nextRightKey;
");
this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields
? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter")
: this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter")));
this.Write(@"
output.hash.col[index] = rightBatch.hash.col[rightBatch.iter];
if (output.Count == Config.DataBatchSize)
{
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
#endregion
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
if (compare <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract." +
"Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" > 0);\r\n currentRightList.Clear();\r\n " +
" }\r\n currentRightKey = rightBatch.key.co" +
"l[rightBatch.iter];\r\n var rightAE = new ActiveEventRi" +
"ght();\r\n rightAE.Populate(ref nextRightTime, generate" +
"dRightBatch, rightBatch.iter);\r\n currentRightList.Add" +
"(rightAE);\r\n }\r\n #endregion\r\n " +
" }\r\n\r\n rightBatch.iter++;\r\n\r\n " +
" #region GoToVisibleRow\r\n /* Inlined version of:\r\n " +
" if (!GoToVisibleRow(rightBatch))\r\n {\r\n " +
" leftBatchDone = false;\r\n right" +
"BatchDone = true;\r\n return;\r\n " +
"}\r\n */\r\n while (rightBatch.iter < rightBat" +
"ch.Count &&\r\n (rightBatch.bitvector.col[rightBatch.iter >" +
"> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n rightB" +
"atch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " +
" rightBatch.iter++;\r\n }\r\n if (rightB" +
"atch.iter == rightBatch.Count)\r\n {\r\n l" +
"eftBatchDone = false;\r\n rightBatchDone = true;\r\n " +
" return;\r\n }\r\n #endregion\r\n\r" +
"\n nextRightTime = rightBatch.vsync.col[rightBatch.iter];\r\n " +
" nextRightKey = rightBatch.key.col[rightBatch.iter];\r\n " +
" }\r\n }\r\n else if (compare < 0)\r\n {\r\n " +
" // process left\r\n #region ProcessLeftStartEdge\r\n " +
" /* Inlined version of:\r\n ProcessLeftStartEdge(\r\n " +
" nextLeftTime,\r\n ref leftBatch.key.col" +
"[leftBatch.iter],\r\n leftBatch[leftBatch.iter],\r\n " +
" leftBatch.hash.col[leftBatch.iter], compare);\r\n */" +
"\r\n {\r\n\r\n\r\n if (currentRightList.Count > 0)\r\n " +
" {\r\n int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey")));
this.Write(";\r\n\r\n Contract.Assert(compare2 >= 0, \"Unexpected compariso" +
"n in sort-ordered join\");\r\n\r\n if (compare2 == 0)\r\n " +
" {\r\n // perform the join\r\n " +
" for (int i = 0; i < currentRightList.Count; i++)\r\n " +
" {\r\n var t = currentRightList[i];\r\n" +
" OutputStartEdge(nextLeftTime > t.Timestamp ? nex" +
"tLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t," +
" leftBatch.hash.col[leftBatch.iter]);\r\n }\r\n " +
" }\r\n else\r\n {\r\n " +
" // clear the right array\r\n curr" +
"entRightList.Clear();\r\n }\r\n }\r\n " +
" }\r\n #endregion\r\n\r\n leftBatch.iter++;\r\n\r\n " +
" #region GoToVisibleRow\r\n /* Inlined version of:\r\n " +
" if (!GoToVisibleRow(leftBatch))\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchDone" +
" = false;\r\n return;\r\n }\r\n " +
" */\r\n while (leftBatch.iter < leftBatch.Count &&\r\n " +
" (leftBatch.bitvector.col[leftBatch.iter >> 6] & (1L << (leftBatch.iter & " +
"0x3f))) != 0 &&\r\n leftBatch.vother.col[leftBatch.iter] >= 0)\r" +
"\n {\r\n leftBatch.iter++;\r\n }\r\n " +
" if (leftBatch.iter == leftBatch.Count)\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchDone = false;\r" +
"\n return;\r\n }\r\n #endregion\r\n\r\n\r" +
"\n #region UpdateNextLeftTime\r\n nextLeftTime = left" +
"Batch.vsync.col[leftBatch.iter];\r\n #endregion\r\n\r\n " +
"nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n\r\n }\r\n e" +
"lse // hot path if larger right side of join matches very few things on the left" +
" side\r\n {\r\n // process right\r\n #region " +
"ProcessRightStartEdge\r\n /* Inlined version of:\r\n P" +
"rocessRightStartEdge(\r\n nextRightTime,\r\n r" +
"ef rightBatch.key.col[rightBatch.iter],\r\n rightBatch.payload." +
"col[rightBatch.iter],\r\n rightBatch.hash.col[rightBatch.iter]," +
" compare);\r\n */\r\n if (currentLeftList.Count > 0)\r\n" +
" {\r\n int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
#region OutputStartEdge
/* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp,
ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */
int index = output.Count++;
output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = nextRightKey;
");
this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields
? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter")
: this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter")));
this.Write(@"
output.hash.col[index] = rightBatch.hash.col[rightBatch.iter];
if (output.Count == Config.DataBatchSize)
{
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
#endregion
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
/*
if (compare <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" > 0);\r\n currentRightList.Clear();\r\n " +
" }\r\n currentRightKey = rightBatch.key.col[rightBatch" +
".iter];\r\n ActiveEvent<");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightAE = new ActiveEvent<");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(">();\r\n rightAE.Populate(ref nextRightTime, ref rightBatch." +
"payload.col[rightBatch.iter]);\r\n currentRightList.Add(rig" +
"htAE);\r\n }\r\n */\r\n #endregion\r\n\r" +
"\n rightBatch.iter++;\r\n\r\n #region GoToVisibleRow\r\n " +
" /* Inlined version of:\r\n if (!GoToVisibleRow(r" +
"ightBatch))\r\n {\r\n leftBatchDone = fals" +
"e;\r\n rightBatchDone = true;\r\n retu" +
"rn;\r\n }\r\n */\r\n while (rightBatc" +
"h.iter < rightBatch.Count &&\r\n (rightBatch.bitvector.col[righ" +
"tBatch.iter >> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n " +
" rightBatch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " +
" rightBatch.iter++;\r\n }\r\n if (rightBatch.it" +
"er == rightBatch.Count)\r\n {\r\n leftBatchDone = " +
"false;\r\n rightBatchDone = true;\r\n return;\r" +
"\n }\r\n #endregion\r\n\r\n #region Update" +
"NextRightTime\r\n /* Inlined version of: UpdateNextRightTime(rightB" +
"atch.vsync.col[rightBatch.iter]); */\r\n nextRightTime = rightBatch" +
".vsync.col[rightBatch.iter];\r\n #endregion\r\n\r\n next" +
"RightKey = rightBatch.key.col[rightBatch.iter];\r\n }\r\n }\r\n }" +
"\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n protected overri" +
"de void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree)
{
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextLeftTime(batch.vsync.col[batch.iter]);
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
nextLeftKey = batch.key.col[batch.iter];
int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(";\r\n if ((compare == 0) && (nextLeftTime <= nextRightTime))\r\n " +
" {\r\n ProcessLeftStartEdge(\r\n nextLeftTime,\r\n " +
" ref batch.key.col[batch.iter],\r\n (");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@")batch, batch.iter,
batch.hash.col[batch.iter], compare);
batch.iter++;
}
else
{
isBatchDone = false;
return;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessRightBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree)
{
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextRightTime(batch.vsync.col[batch.iter]);
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
nextRightKey = batch.key.col[batch.iter];
int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(";\r\n if ((compare == 0) && (nextRightTime <= nextLeftTime))\r\n " +
" {\r\n ProcessRightStartEdge(\r\n nextRightTime,\r" +
"\n ref batch.key.col[batch.iter],\r\n (");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(@")batch,batch.iter,
batch.hash.col[batch.iter], compare);
batch.iter++;
}
else
{
isBatchDone = false;
return;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool GoToVisibleRow<TPayload>(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", TPayload> batch)
{
while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0)
{
batch.iter++;
}
if (batch.iter == batch.Count)
{
return false;
}
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateNextLeftTime(long time)
{
nextLeftTime = time;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateNextRightTime(long time)
{
nextRightTime = time;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessLeftStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" leftBatch, int leftIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " +
" if (currentRightList.Count > 0)\r\n {\r\n int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(@";
Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare == 0)
{
// perform the join
for (int i = 0; i < currentRightList.Count; i++)
{
var t = currentRightList[i];
OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, leftBatch, leftIndex, t, hash);
}
}
else
{
// clear the right array
currentRightList.Clear();
}
}
if (nextLeftRightCompareResult >= 0)
{
// update the left array
if ((currentLeftList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(@" > 0);
currentLeftList.Clear();
}
currentLeftKey = key;
var leftAE = new ActiveEventLeft();
leftAE.Populate(ref start, leftBatch, leftIndex);
currentLeftList.Add(leftAE);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessRightStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" rightBatch, int rightIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " +
" if (currentLeftList.Count > 0)\r\n {\r\n int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, t, rightBatch, rightIndex, hash);
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
if (nextLeftRightCompareResult <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(@" > 0);
currentRightList.Clear();
}
currentRightKey = key;
var rightAE = new ActiveEventRight();
rightAE.Populate(ref start, rightBatch, rightIndex);
currentRightList.Add(rightAE);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddPunctuationToBatch(long start)
{
if (start > lastCTI)
{
lastCTI = start;
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.PunctuationOtherTime;
output.key.col[index] = default;
output[index] = default;
output.hash.col[index] = 0;
output.bitvector.col[index >> 6] |= (1L << (index & 0x3f));
if (output.Count == Config.DataBatchSize) FlushContents();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void OutputStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ActiveEventLeft leftEvent, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" rightBatch, int rightIndex, int hash)\r\n {\r\n int index = output.Count++" +
";\r\n output.vsync.col[index] = start;\r\n output.vother.col[index] = " +
"StreamEvent.InfinitySyncTime;\r\n output.key.col[index] = key;\r\n");
if (this.leftMessageRepresentation.noFields)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent.payload", "rightBatch", "rightIndex")));
this.Write("\r\n");
}
else
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent", "rightBatch", "rightIndex")));
this.Write("\r\n");
}
this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" +
"BatchSize) FlushContents();\r\n }\r\n [MethodImpl(MethodImplOptions.Aggressive" +
"Inlining)]\r\n private void OutputStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@" leftBatch, int leftIndex, ActiveEventRight rightEvent, int hash)
{
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = key;
");
if (this.rightMessageRepresentation.noFields)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent.payload")));
this.Write("\r\n");
}
else
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent")));
this.Write("\r\n");
}
this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" +
"BatchSize) FlushContents();\r\n }\r\n\r\n [DataContract]\r\n private struct Act" +
"iveEventLeft\r\n {\r\n [DataMember]\r\n public long Timestamp;\r\n\r\n " +
" ");
foreach (var f in this.leftFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(ref long timestamp, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n ");
foreach (var f in this.leftFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write(@"
}
public override string ToString()
{
return ""[Timestamp='"" + Timestamp + ""', Payload='"" + ""']"";
}
}
[DataContract]
private struct ActiveEventRight
{
[DataMember]
public long Timestamp;
");
foreach (var f in this.rightFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(ref long timestamp, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n ");
foreach (var f in this.rightFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Timestamp=\'\" + Timestamp + \"\', Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new ClipJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var tm = new TypeMapper(keyType, leftType, rightType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++);
var leftMessageRepresentation = new ColumnarRepresentation(leftType);
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Linq.Expressions;
using System.IO;
using System.Reflection;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n");
string TKey;
var genericTypeNames = new string[] { "A", "B" };
if (keyType.IsAnonymousType())
{
TKey = genericTypeNames[0];
}
else
{
genericTypeNames[0] = string.Empty;
TKey = keyType.GetCSharpSourceSyntax();
}
string TPayload;
var payloadIsAnon = false;
if (payloadType.IsAnonymousType())
{
TPayload = genericTypeNames[1];
payloadIsAnon = true;
}
else
{
genericTypeNames[1] = string.Empty;
TPayload = payloadType.GetCSharpSourceSyntax();
}
var genericParameters = Transformer.GenericParameterList(genericTypeNames);
var genericParameters2 = $"<{TKey}, {TPayload}>";
if (payloadType == typeof(int) || payloadType == typeof(long) || payloadType == typeof(string))
{
genericParameters2 = string.Empty;
}
this.Write("\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.payloadType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n");
if (payloadIsAnon)
{
foreach (var f in this.fields)
{
this.Write(" private PropertyInfo ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property = typeof(");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(").GetProperty(\"");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("\");\r\n");
}
}
this.Write("\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n this.pool = MemoryManager.GetMe" +
"moryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(">(true) as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.payloadType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(@";
this.queryPlanGenerator = queryPlanGenerator;
}
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(this.queryPlanGenerator(previous, this));
}
protected override void FlushContents() { }
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> batch)\r\n {\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> resultBatchUncast; // Need this type to call Get with so the right subtype will" +
" be returned\r\n this.pool.Get(out resultBatchUncast);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" resultBatch = resultBatchUncast as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(";\r\n\r\n resultBatch.CloneFromNoPayload(batch);\r\n\r\n // Get memory pool" +
"s for the payload columns.\r\n\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n this.pool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
if (this.payloadType.CanContainNull())
{
this.Write(" this.pool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write("\r\n var count = batch.Count;\r\n\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.Name));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n\r\n");
}
}
this.Write("\r\n fixed (long* bv = batch.bitvector.col)\r\n {\r\n");
if (this.payloadType.CanBeFixed())
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("* srcpayload = batch.payload.col)\r\n {\r\n");
}
else
{
this.Write("\r\n var srcpayload = batch.payload.col;\r\n\r\n");
}
this.Write("\r\n");
if (this.rowMajor)
{
this.Write(" for (int i = 0; i < count; i++)\r\n {\r\n if ((" +
"bv[i >> 6] & (1L << (i & 0x3f)))==0)\r\n {\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (payloadIsAnon)
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type));
this.Write(") this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property.GetValue(srcpayload[i], null);\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = srcpayload[i].");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
}
this.Write(" }\r\n }\r\n");
}
else
{
this.Write("\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n for (int i = 0; i < count; i++)\r\n {\r\n " +
" if ((bv[i >> 6] & (1L << (i & 0x3f)))==0)\r\n " +
" {\r\n");
if (payloadIsAnon)
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type));
this.Write(") this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property.GetValue(srcpayload[i], null);\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = srcpayload[i].");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
this.Write(" }\r\n }\r\n");
}
this.Write("\r\n");
}
this.Write("\r\n");
if (this.payloadType.CanBeFixed())
{
this.Write(" }\r\n");
}
this.Write(" }\r\n\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n");
}
this.Write(@"
resultBatch.Count = count;
resultBatch.Seal();
this.Observer.OnNext(resultBatch);
batch.Release();
batch.Return();
}
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
}
");
return(this.GenerationEnvironment.ToString());
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedGroupedAfa_{0}", AFASequenceNumber++);
var template = new GroupedAfaTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax()
};
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList1 = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, searc, "i");
edgeList1.Add(edgeInfo);
}
}
template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, searc, "i");
edgeList2.Add(edgeInfo);
}
}
template.newActivationInfo.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree;
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private Tuple <Type, string> GenerateInternal <TKey, TLeft, TRight, TResult>(int numParameters, Expression expression)
{
string errorMessages = null;
try
{
var expandedCode = TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(this.keyType, this.leftType, this.rightType, this.resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
if (expression != null)
{
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(expression));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for a binary operator is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var types = new List <Type> {
this.keyType, this.leftType
};
if (numParameters > 2)
{
types.Add(this.rightType);
}
if (numParameters == 4)
{
types.Add(this.resultType);
}
var realClassName = this.className.AddNumberOfNecessaryGenericArguments(types.ToArray());
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = this.keyType.GetAnonymousTypes();
list.AddRange(this.leftType.GetAnonymousTypes());
if (numParameters > 2)
{
list.AddRange(this.rightType.GetAnonymousTypes());
}
if (numParameters == 4)
{
list.AddRange(this.resultType.GetAnonymousTypes());
}
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TInnerKey, int> > hashComparer,
Expression <Func <TSource, TInnerKey> > keySelector,
bool nested)
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
string generatedClassName = string.Format("GeneratedGroupStreamable_{0}", GroupStreamableSequenceNumber++);
string transformedKeySelectorAsString;
MyFieldInfo swingingField = default;
if (typeOfTInnerKey.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments.Select(arg => arg.ExpressionToCSharp()));
}
else
{
var body = keySelector.Body;
var singleFieldProjection = body as MemberExpression;
if (!nested && singleFieldProjection != null)
{
if (singleFieldProjection.Expression is ParameterExpression dereferencedObject)
{
Contract.Assume(dereferencedObject == keySelector.Parameters.ElementAt(0));
// then can just swing a pointer to set the key of the result message
var f = singleFieldProjection.Member;
var sourceMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var fieldToSwingFrom = sourceMessageRepresentation.Fields[f.Name];
swingingField = fieldToSwingFrom;
transformedKeySelectorAsString = swingingField.Name + "_col[i]";
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
var inlinedHashCodeComputation = hashComparer.Inline("key");
var template = new GroupTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
transformedKeySelectorAsString,
inlinedHashCodeComputation,
nested)
{
swingingField = swingingField,
payloadMightBeNull = typeOfTSource.CanContainNull()
};
if (!nested && Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
template.transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name;
template.vectorHashCodeInitialization = string.Format("resultBatch.hash = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName);
template.swingingHashColumn = true;
}
template.fields = new ColumnarRepresentation(typeOfTSource).AllFields;
template.innerKeyFieldRepresentation = new ColumnarRepresentation(typeOfTInnerKey);
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Group is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator has has no support for ECQ, start edge only\r\n/// </" +
"summary>\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n private long hop;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write("\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write("\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write("\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool> keyComparerEquals;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", int> keyComparerGetHashCode;\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" [DataMember]\r\n private FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> heldAggregates;\r\n");
}
else
{
this.Write(" [DataMember]\r\n private HeldState currentState;\r\n");
}
this.Write("\r\n [DataMember]\r\n private long lastSyncTime = long.MinValue;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate,\r\n long hop)\r\n : base(stream, observer)\r\n {\r\n " +
" this.aggregate = aggregate;\r\n this.queryPlanGenerator = queryPlanGenerat" +
"or;\r\n this.hop = hop;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write("\r\n initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write("\r\n accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write("\r\n computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write(@"
this.keyComparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEquals = this.keyComparer.GetEqualsExpr().Compile();
this.keyComparerGetHashCode = this.keyComparer.GetGetHashCodeExpr().Compile();
this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" var generator = this.keyComparer.CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.Query" +
"Container);\r\n this.heldAggregates = generator.Invoke();\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n\r\n protected override void DisposeSta" +
"te() => this.batch.Free();\r\n\r\n public override int CurrentlyBufferedOutputCou" +
"nt => this.batch.Count;\r\n\r\n public override int CurrentlyBufferedInputCount =" +
"> ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "0" : "this.heldAggregates.Count"));
this.Write(";\r\n\r\n public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n var count = generatedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@" var col_key = generatedBatch.key.col;
fixed (long* col_vsync = generatedBatch.vsync.col)
fixed (long* col_vother = generatedBatch.vother.col)
fixed (int* col_hash = generatedBatch.hash.col)
fixed (long* col_bv = generatedBatch.bitvector.col)
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == StreamEvent.PunctuationOtherTime)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime;
");
if (!this.isUngrouped)
{
this.Write(" this.batch.key.col[c] = default;\r\n");
}
this.Write(@" this.batch[c] = default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
");
if (this.isUngrouped)
{
this.Write(@" if (currentState != null) // there exists earlier state
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = currentState.timestamp + this.hop;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.hash.col[c] = 0;\r\n this.batch" +
".Count++;\r\n if (this.batch.Count == Config.DataBatchSize) Flu" +
"shContents();\r\n currentState = null;\r\n }\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = this.lastSyncTime;
this.batch.vother.col[c] = this.lastSyncTime + this.hop;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clear();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n " +
" this.lastSyncTime = syncTime;\r\n }\r\n\r\n");
if (this.isUngrouped)
{
this.Write(" if (currentState == null)\r\n {\r\n currentStat" +
"e = new HeldState();\r\n currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(";\r\n currentState.timestamp = syncTime;\r\n }\r\n\r\n " +
" currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]")));
this.Write(";\r\n");
}
else
{
this.Write(@" // Need to retrieve the key from the dictionary
var currentKey = col_key[i];
var currentHash = col_hash[i];
if (!this.heldAggregates.Lookup(currentKey, currentHash, out int index))
this.heldAggregates.Insert(ref index, currentKey, ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(");\r\n\r\n ref var entry = ref this.heldAggregates.entries[index];\r\n " +
" entry.value = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("entry.value", "col_vsync[i]")));
this.Write(";\r\n");
}
this.Write(" }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n " +
" public void OnPunctuation(long syncTime)\r\n {\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Handle time moving forward\r\n if (syncTime > this.lastSyncTime" +
")\r\n {\r\n");
if (this.isUngrouped)
{
this.Write(@" if (currentState != null) // need to send interval event
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = currentState.timestamp + this.hop;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n\r\n this.batch.hash.col[c] = 0;\r\n this.batch.Count" +
"++;\r\n if (this.batch.Count == Config.DataBatchSize) FlushContents" +
"();\r\n currentState = null;\r\n }\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = this.lastSyncTime;
this.batch.vother.col[c] = this.lastSyncTime + this.hop;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clear();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n this" +
".lastSyncTime = syncTime;\r\n }\r\n }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("Generated{1}AfaMultiEventList_{0}", AFASequenceNumber++, typeof(TKey).Equals(typeof(Empty)) ? string.Empty : "Grouped");
var template = new AfaMultiEventListTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax()
};
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
template.payloadIsAnon = typeof(TPayload).IsAnonymousTypeName();
template.payloadHasNoFields = payloadRepresentation.noFields;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = sourceNodeNumber,
TargetNode = targetNodeNumber,
fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber),
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList.Add(multiEdgeInfo);
continue;
}
if (edge is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, singleArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
if (edge is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNodeNumber, listArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
}
template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var eps = EpsilonClosure(stream.afa, targetNode);
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = startState,
TargetNode = targetNode,
fromStartState = true,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList2.Add(multiEdgeInfo);
continue;
}
if (arc is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, singleArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
if (arc is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNode, listArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
}
template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(Queue <>));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator that uses a full-fledged priority queue as ECQ\r\n///" +
" </summary>\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n\r\n [DataContract]\r\n class EcqState\r\n {\r\n [DataMembe" +
"r]\r\n public long timestamp;\r\n [DataMember]\r\n public FastDic" +
"tionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive> states;\r\n }\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> deaccumulate;\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> difference;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool> keyComparerEquals;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", int> keyComparerGetHashCode;\r\n\r\n [DataMember]\r\n private FastDictionary2<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> aggregateByKey;\r\n [DataMember]\r\n private FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> heldAggregates;\r\n [DataMember]\r\n private SortedDictionary<long" +
", EcqState> ecq;\r\n [DataMember]\r\n private long lastSyncTime = long.MinValu" +
"e;\r\n\r\n private HeldState currentState;\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentKey;\r\n private int currentHash;\r\n\r\n private Func<FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>> stateDictGenerator;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" +
"ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" deaccumulate = aggregate.Deaccumulate().Compile();\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" difference = aggregate.Difference().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write(@"
this.keyComparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEquals = this.keyComparer.GetEqualsExpr().Compile();
this.keyComparerGetHashCode = this.keyComparer.GetGetHashCodeExpr().Compile();
this.stateDictGenerator = this.keyComparer.CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream." +
"Properties.QueryContainer);\r\n\r\n this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n this.aggregateByKey = this.keyCompare" +
"r.CreateFastDictionary2Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Prope" +
"rties.QueryContainer).Invoke();\r\n this.heldAggregates = this.keyComparer." +
"CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke();
this.ecq = new SortedDictionary<long, EcqState>();
}
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
this.batch.Allocate();
}
protected override void DisposeState() => this.batch.Free();
public override int CurrentlyBufferedOutputCount => this.batch.Count;
public override int CurrentlyBufferedInputCount => this.aggregateByKey.Count + this.ecq.Values.Select(o => o.states.Count).Sum();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n this.batch.iter = generatedBatch.iter;\r\n\r\n var count = genera" +
"tedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n fixed (long* col_vsync = generatedBatch.vsync.col)\r\n fixed (long" +
"* col_vother = generatedBatch.vother.col)\r\n fixed (int* col_hash = genera" +
"tedBatch.hash.col)\r\n fixed (long* col_bv = generatedBatch.bitvector.col)\r" +
"\n for (int i = 0; i < count; i++)\r\n {\r\n if ((col_bv[i >" +
"> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n if (col_vother[" +
"i] == long.MinValue)\r\n {\r\n // We have found a " +
"row that corresponds to punctuation\r\n OnPunctuation(col_vsync" +
"[i]);\r\n\r\n int c = this.batch.Count;\r\n this" +
".batch.vsync.col[c] = col_vsync[i];\r\n this.batch.vother.col[c" +
"] = long.MinValue;\r\n this.batch.key.col[c] = default;\r\n " +
" this.batch[c] = default;\r\n this.batch.hash.col[" +
"c] = 0;\r\n this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f" +
");\r\n this.batch.Count++;\r\n if (this.batch." +
"Count == Config.DataBatchSize) FlushContents();\r\n }\r\n " +
" continue;\r\n }\r\n\r\n var syncTime = col_vsync[i];\r\n " +
" var key_i = generatedBatch.key.col[i];\r\n\r\n HeldState heldState;" +
"\r\n bool cachedState = false;\r\n\r\n // Handle time moving for" +
"ward\r\n if (syncTime > this.lastSyncTime)\r\n {\r\n " +
" /* Issue start edges for held aggregates */\r\n if (currentStat" +
"e != null && this.heldAggregates.Count == 1)\r\n {\r\n " +
" // there is just one held aggregate, and currentState is set\r\n " +
" // so currentState has to be the held aggregate\r\n cach" +
"edState = true;\r\n\r\n if (currentState.state.active > 0)\r\n " +
" {\r\n int c = this.batch.Count;\r\n " +
" this.batch.vsync.col[c] = currentState.timestamp;\r\n " +
" this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;\r\n " +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.iter = generatedBatch.iter;
FlushContents();
this.batch.iter = generatedBatch.iter;
}
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash);
currentState = null;
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
}
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))" +
"\r\n {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Update aggregate
heldState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries" +
"[iter].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n " +
" if (this.batch.Count == Config.DataBatchSize) FlushContents();\r\n " +
" }\r\n else\r\n " +
" {\r\n // remove from aggregateByKey\r\n " +
" this.aggregateByKey.Remove(ecqState.states.entries[iter].k" +
"ey);\r\n currentState = null;\r\n " +
" }\r\n }\r\n else\r\n " +
" {\r\n if (cachedState)\r\n " +
" {\r\n cachedState = false;\r\n " +
" if (heldState != currentState)\r\n " +
" {\r\n this.heldAggregates.Clear();\r\n " +
" currentState = null;\r\n " +
" int index;\r\n this.heldAggregates.Lookup(" +
"ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n " +
" else\r\n {\r\n " +
" int index;\r\n this.heldAggregates.Loo" +
"kup(ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n\r\n " +
" // Update timestamp\r\n heldState.timestamp =" +
" ecqState.timestamp;\r\n }\r\n }\r\n\r\n " +
" // Since sync time changed, set this.lastSyncTime\r\n this.lastSy" +
"ncTime = syncTime;\r\n }\r\n\r\n if (currentState == null || cur" +
"rentHash != col_hash[i] || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "key_i")));
this.Write(@")
{
if (cachedState)
{
cachedState = false;
this.heldAggregates.Clear();
}
// Need to retrieve the key from the dictionary
currentKey = key_i;
currentHash = col_hash[i];
int index;
if (!this.heldAggregates.Lookup(currentKey, currentHash, out index))
{
// First time group is active for this time
int aggindex;
if (!this.aggregateByKey.Lookup(currentKey, currentHash, out aggindex))
{
// New group. Create new state
currentState = new HeldState();
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
this.aggregateByKey.Insert(currentKey, currentState, currentHash);
// No output because initial state is empty
}
else
{
// Update instance of key in case consumer tracks lifetime of the key object.
// Otherwise it may live past the Window lifetime.
this.aggregateByKey.entries[aggindex].key = currentKey;
currentState = this.aggregateByKey.entries[aggindex].value;
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
this.heldAggregates.Insert(ref index, currentKey, currentState);
}
else
{
// read new currentState from _heldAgg index
currentState = this.heldAggregates.entries[index].value;
}
}
else
{
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
if (col_vsync[i] < col_vother[i]) // insert event
{
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(@";
currentState.state.active++;
// Update ECQ
if (col_vother[i] < StreamEvent.InfinitySyncTime)
{
EcqState eState;
int index;
if (this.ecq.Count > 0)
{
if (!this.ecq.TryGetValue(col_vother[i], out eState))
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
this.ecq.Add(col_vother[i], eState);
}
else
{
if (!eState.states.Lookup(key_i, col_hash[i], out index))
{
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
}
}
}
else
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(" });\r\n this.ecq.Add(col_vother[i], eState);\r\n " +
" }\r\n\r\n eState.states.entries[index].value.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("eState.states.entries[index].value.state", "col_vsync[i]")));
this.Write(";\r\n eState.states.entries[index].value.active++;\r\n " +
" }\r\n }\r\n else // is a retraction\r\n {\r\n " +
" currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(deaccumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(";\r\n currentState.state.active--;\r\n }\r\n }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n\r" +
"\n public void OnPunctuation(long syncTime)\r\n {\r\n HeldState heldStat" +
"e;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@"
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
/* Issue start edges for held aggregates */
if (currentState != null && this.heldAggregates.Count == 1)
{
// there is just one held aggregate, and currentState is set
// so currentState has to be the held aggregate
if (currentState.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n th" +
"is.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key);
currentState = null;
}
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))\r\n " +
" {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[iter].k" +
"ey;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n if (this." +
"batch.Count == Config.DataBatchSize) FlushContents();\r\n }\r\n\r\n" +
" // Update aggregate\r\n heldState.state.sta" +
"te = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
// remove from aggregateByKey
this.aggregateByKey.Remove(ecqState.states.entries[iter].key);
currentState = null;
}
}
else
{
int index;
this.heldAggregates.Lookup(ecqState.states.entries[iter].key, out index);
this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, heldState);
}
// Update timestamp
heldState.timestamp = ecqState.timestamp;
}
}
// Since sync time changed, set this.lastSyncTime
this.lastSyncTime = syncTime;
}
}
protected override void UpdatePointers()
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>.IteratorStart;\r\n while (this.heldAggregates.Iterate(ref iter1" +
"))\r\n {\r\n int iter2;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = this.heldAggregates.entries[iter1].key;
bool found = this.aggregateByKey.Lookup(key, out iter2);
if (!found) throw new InvalidOperationException();
this.heldAggregates.entries[iter1].value = this.aggregateByKey.entries[iter2].value;
}
}
}
");
return(this.GenerationEnvironment.ToString());
}
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeofTInnerResult = typeof(TInnerResult);
var typeofTResult = typeof(TResult);
string expandedCode;
List <Assembly> assemblyReferences;
int numberOfGenericParameters;
// inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters);
var generatedClassName = string.Format(
"UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}",
typeOfTOuterKey.GetValidIdentifier(),
typeOfTInnerKey.GetValidIdentifier(),
typeofTInnerResult.GetValidIdentifier(),
typeofTResult.GetValidIdentifier(),
UngroupSequenceNumber++);
var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>();
var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult);
var resultRepresentation = new ColumnarRepresentation(typeofTResult);
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
// Leave the key parameter to the selector unchanged, so no substitution for parameters[0]
Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation()
{
BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation,
})
};
var result = SelectTransformer.Transform(resultSelector, parameterSubsitutions, resultRepresentation);
if (result.Error)
{
return(Tuple.Create((Type)null, errorMessages));
}
;
var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult)
{
innerResultRepresentation = innerResultRepresentation,
swingingFields = result.SwingingFields,
computedFields = result.ComputedFields,
unassignedFields = result.UnmentionedFields,
keyParameter = resultSelector.Parameters.First()
};
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
// input messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>());
// output messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>());
// memory pool
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
numberOfGenericParameters = template.numberOfGenericParameters;
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult))
{
leftDuration = stream.Left.Properties.ConstantDurationLength.Value,
rightDuration = stream.Right.Properties.ConstantDurationLength.Value
};
var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
var resultMessageRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (template.keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => $"keyComparerEquals({left}, {right})";
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>());
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectManyStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string expandedCode;
string errorMessages = null;
try
{
generatedClassName = $"SelectMany_{sequenceNumber++}";
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
var template = new SelectManyTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
var selector = stream.Selector;
var payloadParameter = selector.Parameters.ElementAt(payloadParameterIndex);
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.resultFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
expandedCode = string.Empty;
Expression transformedSelector = selector;
// No substitutions are made for the start edge parameter or key parameter. Both just remain in the
// body of the result selector and are set as local variables in the generated code.
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
var tuple = OptimizeSelectMany(selector.Body);
if (tuple != null)
{
template.enumerableRepeatSelector = true;
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var pseudoLambdaParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
var pseudoLambdaIndex = 0;
if (stream.HasKey)
{
pseudoLambdaParameters[pseudoLambdaIndex++] = selector.Parameters[keyParameterIndex];
}
pseudoLambdaParameters[pseudoLambdaIndex] = payloadParameter;
var pseudoLambda = Expression.Lambda(tuple.Item1, pseudoLambdaParameters);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
Tuple.Create(payloadParameter, new SelectParameterInformation()
{
BatchName = "batch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
})
};
var projectionResult = SelectTransformer.Transform(pseudoLambda, parameterSubstitutions, template.resultPayloadRepresentation, true, stream.HasStartEdge);
template.computedFields = projectionResult.ComputedFields;
template.projectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.useEnumerator = false;
var loopCounter = tuple.Item2;
var newParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
newParameters[0] = payloadParameter;
if (stream.HasKey)
{
newParameters[1] = selector.Parameters[keyParameterIndex];
}
var loopCounterLambda = Expression.Lambda(loopCounter, payloadParameter);
var transformedLoopCounter = Extensions.TransformFunction <TKey, TPayload>(loopCounterLambda, 0);
template.loopCounter = transformedLoopCounter.Body.ExpressionToCSharp();
// REVIEW: Alternative: use Inline to replace occurrences of the key parameter
// with "batch.key.col[i]".
if (stream.HasKey)
{
template.keyParameterName = selector.Parameters[keyParameterIndex].Name;
}
}
else
{
transformedSelector = Extensions.TransformFunction <TKey, TPayload>(stream.Selector, payloadParameterIndex).Body;
if (transformedSelector == null)
{
template.useEnumerator = true;
template.transformedSelectorAsSource = stream.Selector.ExpressionToCSharp();
}
else
{
var tuple2 = OptimizeSelectMany(transformedSelector);
if (tuple2 != null)
{
template.useEnumerator = false;
template.loopCounter = tuple2.Item2.ExpressionToCSharp();
template.transformedSelectorAsSource = tuple2.Item1.ExpressionToCSharp();
}
else
{
template.useEnumerator = true;
template.transformedSelectorAsSource = transformedSelector.ExpressionToCSharp();
}
}
}
template.StartEdgeParameterName = stream.HasStartEdge ? selector.Parameters.ElementAt(0).Name : null;
template.hasKey = stream.HasKey;
expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var container = stream.Properties.QueryContainer;
string generatedClassName = string.Format("Aggregate_{0}", sequenceNumber++);
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
AggregateTemplate template;
switch (pipeType)
{
case AggregatePipeType.StartEdge:
template = new SnapshotWindowStartEdgeTemplate(generatedClassName);
break;
case AggregatePipeType.PriorityQueue:
template = new SnapshotWindowPriorityQueueTemplate(generatedClassName);
break;
case AggregatePipeType.Tumbling:
template = new SnapshotWindowTumblingTemplate(generatedClassName);
break;
case AggregatePipeType.Sliding:
template = new SnapshotWindowSlidingTemplate(generatedClassName);
break;
case AggregatePipeType.Hopping:
template = new SnapshotWindowHoppingTemplate(
generatedClassName,
((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString());
break;
default:
Contract.Assert(false, "case meant to be exhaustive");
throw new InvalidOperationException("case meant to be exhaustive");
}
template.isUngrouped = typeof(TKey) == typeof(Empty);
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
template.inputFields = new ColumnarRepresentation(inputType).AllFields;
template.outputFields = new ColumnarRepresentation(outputType).AllFields;
assemblyReferences = new List <Assembly>();
#region Key Comparer
IEqualityComparerExpression <TKey> keyComparer;
keyComparer = stream.Properties.KeyEqualityComparer;
var equalsExpression = keyComparer.GetEqualsExpr();
var getHashcodeExpression = keyComparer.GetGetHashCodeExpr();
template.inlinedKeyComparerEquals =
(left, right) =>
$"({equalsExpression.Inline(left, right)})";
template.inlinedKeyComparerGetHashCode =
(x) =>
$"({getHashcodeExpression.Inline(x)}/* inlined GetHashCode */)";
if (keyType.IsAnonymousType())
{
template.inlinedKeyComparerEquals =
(left, right) => $"keyComparerEquals({left}, {right})";
template.inlinedKeyComparerGetHashCode =
(x) => $"keyComparerGetHashCode({x})";
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression));
#endregion
#region Aggregate Functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>();
var foo = Transform(newExpression, outputBatchType);
template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType())
{
// second disjunct is because if we aren't inlining the computeResult function and
// the output type is anonymous, calling the compiled computeResult function returns
// a value of the anonymous type and since the generated operator represents the anonymous
// type as a generic parameter, it can't use the "field" (i.e., property) names to
// get the individual pieces to assign to each column of the output message.
errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ");";
}
}
#endregion
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>());
if (container != null)
{
assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">\r\n{\r\n private readonly MemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> pool;\r\n\r\n [SchemaSerialization]\r\n private readonly Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool>> keyComparer;\r\n [SchemaSerialization]\r\n private readonly Expression" +
"<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", bool>> leftComparer;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@" output;
[DataMember]
private FastMap<LeftInterval> leftIntervalMap = new FastMap<LeftInterval>();
[DataMember]
private FastMap<LeftEdge> leftEdgeMap = new FastMap<LeftEdge>();
[DataMember]
private RemovableEndPointHeap leftEndPointHeap;
[DataMember]
private long nextLeftTime = long.MinValue;
[DataMember]
private long nextRightTime = long.MinValue;
[DataMember]
private long currTime = long.MinValue;
private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator;
");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n [Obsolete(\"Used only by serialization. Do not call directly.\")]\r\n publ" +
"ic ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> observer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyEqualityComparer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@"> leftComparer,
Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator)
: base(stream, observer)
{
this.queryPlanGenerator = queryPlanGenerator;
this.keyComparer = keyEqualityComparer.GetEqualsExpr();
this.leftComparer = leftComparer.GetEqualsExpr();
this.leftEndPointHeap = new RemovableEndPointHeap();
this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
GetOutputBatch();
this.Write(@" }
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count + leftIntervalMap.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
this.Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
protected override void DisposeState() => this.output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericLeftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericRightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool le" +
"ftBatchFree, out bool rightBatchFree)\r\n {\r\n var leftBatch = genericLef" +
"tBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(";\r\n var rightBatch = genericRightBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
leftBatchFree = rightBatchFree = true;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
nextLeftTime = leftBatch.vsync.col[leftBatch.iter];
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
nextRightTime = rightBatch.vsync.col[rightBatch.iter];
while (true)
{
if (nextLeftTime <= nextRightTime)
{
UpdateTime(nextLeftTime);
/*
ProcessLeftEvent(
nextLeftTime,
leftBatch.vother.col[leftBatch.iter],
ref leftBatch.key.col[leftBatch.iter],
leftBatch[leftBatch.iter],
leftBatch.hash.col[leftBatch.iter]);
*/
");
ProcessLeftEvent("nextLeftTime", "leftBatch.vother.col[leftBatch.iter]", "leftBatch.key.col[leftBatch.iter]", "leftBatch", "leftBatch.iter", "leftBatch.hash.col[leftBatch.iter]");
this.Write(@"
leftBatch.iter++;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
nextLeftTime = leftBatch.vsync.col[leftBatch.iter];
}
else
{
UpdateTime(nextRightTime);
ProcessRightEvent(
nextRightTime,
rightBatch.vother.col[rightBatch.iter],
ref rightBatch.key.col[rightBatch.iter],
rightBatch.hash.col[rightBatch.iter]);
rightBatch.iter++;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
nextRightTime = rightBatch.vsync.col[rightBatch.iter];
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
nextLeftTime = batch.vsync.col[batch.iter];
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
UpdateTime(nextLeftTime);
/*
ProcessLeftEvent(
nextLeftTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch[batch.iter],
batch.hash.col[batch.iter]);
*/
");
ProcessLeftEvent("nextLeftTime", "batch.vother.col[batch.iter]", "batch.key.col[batch.iter]", "batch", "batch.iter", "batch.hash.col[batch.iter]");
this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOption" +
"s.AggressiveInlining)]\r\n protected override void ProcessRightBatch(StreamMess" +
"age<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
nextRightTime = batch.vsync.col[batch.iter];
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
UpdateTime(nextRightTime);
ProcessRightEvent(
nextRightTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch.hash.col[batch.iter]);
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool GoToVisibleRow<TPayload>(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", TPayload> batch)
{
while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0)
{
batch.iter++;
}
return (batch.iter != batch.Count);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateTime(long time)
{
if (time != currTime)
{
currTime = time;
ReachTime();
}
}
/*
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessLeftEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" payload, int hash)\r\n {\r\n if (start < end)\r\n {\r\n // R" +
"ow is a start edge or interval.\r\n bool isInterval = end < StreamEvent" +
".InfinitySyncTime;\r\n if (isInterval)\r\n {\r\n " +
"bool isFullyOutputtable = nextRightTime >= end;\r\n if (isFullyOutp" +
"uttable)\r\n {\r\n // Output full interval.\r\n " +
" AddToBatch(start, end, ref key, ref payload, hash);\r\n " +
" }\r\n else\r\n {\r\n // Insert in" +
"to map to remember interval.\r\n int mapIndex = leftIntervalMap" +
".Insert(hash);\r\n\r\n // Insert into heap to schedule removal at" +
" endpoint.\r\n int heapIndex = leftEndPointHeap.Insert(end, map" +
"Index);\r\n\r\n // Set value in map, also remembering heap\'s inde" +
"x.\r\n leftIntervalMap.Values[mapIndex].Initialize(start, ref k" +
"ey, ref payload, heapIndex);\r\n\r\n // Output start edge.\r\n " +
" AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payl" +
"oad, hash);\r\n }\r\n }\r\n else\r\n {\r\n" +
" int index = leftEdgeMap.Insert(hash);\r\n leftEdgeM" +
"ap.Values[index].Populate(start, ref key, ref payload);\r\n\r\n // Ou" +
"tput start edge.\r\n AddToBatch(start, StreamEvent.InfinitySyncTime" +
", ref key, ref payload, hash);\r\n }\r\n }\r\n else if (end =" +
"= StreamEvent.PunctuationOtherTime)\r\n {\r\n AddPunctuationToBatc" +
"h(start);\r\n }\r\n else\r\n {\r\n // Row is an end edge" +
".\r\n var leftEvents = leftEdgeMap.Find(hash);\r\n int index;\r" +
"\n while (leftEvents.Next(out index))\r\n {\r\n " +
"var leftEdge = leftEdgeMap.Values[index];\r\n var leftKey = leftEdg" +
"e.Batch.key.col[leftEdge.Index];\r\n if (AreSame(end, ref key, ref " +
"payload, ref leftEdgeMap.Values[index]))\r\n {\r\n " +
" // Output end edge.\r\n AddToBatch(start, end, ref key, ref pa" +
"yload, hash);\r\n\r\n // Remove from leftMap.\r\n " +
" leftEvents.Remove();\r\n break;\r\n }\r\n " +
" }\r\n }\r\n }\r\n*/\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlini" +
"ng)]\r\n private void ProcessRightEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key, int hash)
{
if (end == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(start);
return;
}
else if (start >= end)
{
// Row is an end edge, which we don't care about because the start edge would have already
// removed all joining left events.
return;
}
// Mark any matching left intervals as no longer active.
int index;
var leftIntervals = leftIntervalMap.Find(hash);
while (leftIntervals.Next(out index))
{
long leftStart = leftIntervalMap.Values[index].Start;
var leftInterval = leftIntervalMap.Values[index];
var leftIntervalKey = leftInterval.Key;
if (leftStart < start && ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftIntervalKey")));
this.Write(@")
{
// Output end edge.
/*
AddToBatch(
start,
leftStart,
ref leftIntervalMap.Values[index].Key,
ref leftIntervalMap.Values[index].Payload,
hash);
*/
");
AddToBatch("start", "leftStart", "leftIntervalKey", "leftIntervalMap.Values[index]", "hash");
this.Write(@"
// Remove from heap and map.
leftEndPointHeap.Remove(leftIntervalMap.Values[index].HeapIndex);
leftIntervals.Remove();
}
}
// Remove any matching left edges.
var leftEdges = leftEdgeMap.Find(hash);
while (leftEdges.Next(out index))
{
var leftEdge = leftEdgeMap.Values[index];
long leftStart = leftEdge.Start;
var leftEdgeKey = leftEdge.Key;
if (leftStart < start && ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftEdgeKey")));
this.Write(@")
{
// Output end edge.
/*
AddToBatch(
start,
leftStart,
ref leftEdgeMap.Values[index].Key,
ref leftEdgeMap.Values[index].Payload,
hash);
*/
");
AddToBatch("start", "leftStart", "leftEdgeKey", "leftEdge", "hash");
this.Write(@"
// Remove left event.
leftEdges.Remove();
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ReachTime()
{
// Carry-out all interval endpoints for left intervals that end prior or at new current time.
long endPointTime;
int index;
while (leftEndPointHeap.TryGetNextInclusive(currTime, out endPointTime, out index))
{
// Output end edge.
/*
AddToBatch(
endPointTime,
leftIntervalMap.Values[index].Start,
ref leftIntervalMap.Values[index].Key,
ref leftIntervalMap.Values[index].Payload,
leftIntervalMap.GetHash(index));
*/
var leftInterval = leftIntervalMap.Values[index];
var leftIntervalKey = leftInterval.Key;
var leftIntervalStart = leftInterval.Start;
var leftIntervalHash = leftIntervalMap.GetHash(index);
");
AddToBatch("endPointTime", "leftIntervalStart", "leftIntervalKey", "leftInterval", "leftIntervalHash");
this.Write(@"
// Remove from leftMap.
leftIntervalMap.Remove(index);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddPunctuationToBatch(long start)
{
if (start > lastCTI)
{
lastCTI = start;
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.PunctuationOtherTime;
output.key.col[index] = default;
");
foreach (var f in this.leftFields)
{
if (f.OptimizeString())
{
this.Write(" output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(string.Empty);\r\n");
}
else
{
this.Write(" output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[index] = default;\r\n");
}
}
this.Write(@" output.hash.col[index] = 0;
output.bitvector.col[index >> 6] |= (1L << (index & 0x3f));
if (output.Count == Config.DataBatchSize) FlushContents();
}
}
/*
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddToBatch(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, int hash)
{
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = end;
output.key.col[index] = key;
output[index] = payload;
output.hash.col[index] = hash;
if (output.Count == Config.DataBatchSize)
{
this.Observer.OnNext(output);
pool.Get(out output);
output.Allocate();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool AreSame(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, ref LeftEdge active)
{
return start == active.Start && keyComparerEquals(key, active.Key) && leftComparerEquals(payload, active.Payload);
}
*/
protected override void FlushContents()
{
if (output.Count == 0) return;
this.Observer.OnNext(output);
");
GetOutputBatch();
this.Write(" }\r\n");
if (!noFields && !this.leftType.GetTypeInfo().IsValueType)
{
this.Write(" [DataContract]\r\n private struct ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write("\r\n {\r\n");
foreach (var f in this.leftFields)
{
this.Write(" [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
this.Write(" }\r\n");
}
this.Write("/*\r\n [DataContract]\r\n private struct LeftInterval\r\n {\r\n [DataMemb" +
"er]\r\n public long Start;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" Payload;\r\n [DataMember]\r\n public int HeapIndex;\r\n\r\n [Method" +
"Impl(MethodImplOptions.AggressiveInlining)]\r\n public void Initialize(long" +
" start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, int heapIndex)
{
Start = start;
Key = key;
Payload = payload;
HeapIndex = heapIndex;
}
public override string ToString()
{
return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""', HeapIndex="" + HeapIndex + ""]"";
}
}
*/
[DataContract]
private struct LeftInterval
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public int HeapIndex;\r\n [DataMember]\r" +
"\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Initialize(long start, ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(" batch, int index, int heapIndex)\r\n {\r\n Start = start;\r\n " +
" Key = batch.key.col[index];\r\n");
foreach (var f in this.leftFields)
{
this.Write(" this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n");
}
this.Write(@" HeapIndex = heapIndex;
}
public override string ToString()
{
//return string.Format(""[Start={0}, Key='{1}', Payload='{2}', HeapIndex={3}"", Start, Key, Batch[Index], HeapIndex);
System.Text.StringBuilder sb = new System.Text.StringBuilder();
sb.AppendFormat(""[Start={0}"", Start);
sb.AppendFormat("", Key='{0}'"", Key);
");
foreach (var f in this.leftFields)
{
this.Write(" sb.AppendFormat(\", ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("={0}, \", Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(");\r\n ");
}
this.Write(@" sb.AppendFormat("", Key='{0}'"", HeapIndex);
sb.Append(""]"");
return sb.ToString();
}
}
/*
[DataContract]
private struct LeftEdge
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Populate(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload)
{
Start = start;
Key = key;
Payload = payload;
}
public override string ToString()
{
return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""]"";
}
}
*/
[DataContract]
private struct LeftEdge
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Populate(long start, ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(" batch, int index)\r\n {\r\n Start = start;\r\n Key = batc" +
"h.key.col[index];\r\n");
foreach (var f in this.leftFields)
{
this.Write(" this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n");
}
this.Write(@" }
public override string ToString()
{
System.Text.StringBuilder sb = new System.Text.StringBuilder();
sb.AppendFormat(""[Start={0}"", Start);
sb.AppendFormat("", Key='{0}'"", Key);
");
foreach (var f in this.leftFields)
{
this.Write(" sb.AppendFormat(\", ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("={0}, \", Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(");\r\n ");
}
this.Write(" sb.Append(\"]\");\r\n return sb.ToString();\r\n }\r\n }\r" +
"\n}\r\n\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
template.TKey = keyType.GetCSharpSourceSyntax();
template.TLeft = leftType.GetCSharpSourceSyntax();
template.TRight = rightType.GetCSharpSourceSyntax();
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.className = string.Format("GeneratedIncreasingOrderEquiJoin_{0}", IOOEJSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(keyType, resultType),
template.TKeyTResultGenericParameters);
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(template.keyType, template.resultType),
template.TKeyTResultGenericParameters);
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
