internal MemoryPoolTemplate(ColumnarRepresentation keyRepresentation, ColumnarRepresentation payloadRepresentation)
{
this.assemblyReferences = new List <Assembly>();
var keyType = keyRepresentation == null ? typeof(Empty) : keyRepresentation.RepresentationFor;
Contract.Assume(Transformer.IsValidKeyType(keyType));
this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType));
this.keyType = keyType;
#region Decompose TPayload into columns
var payloadType = payloadRepresentation.RepresentationFor;
this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType));
this.payloadType = payloadType;
var payloadTypes = payloadRepresentation.AllFields.Select(f => f.Type).Where(t => !t.MemoryPoolHasGetMethodFor());
this.types = new HashSet <Type>(payloadTypes.Distinct());
this.assemblyReferences.AddRange(this.types.SelectMany(t => Transformer.AssemblyReferencesNeededFor(t)));
#endregion
this.generatedClassName = Transformer.GetMemoryPoolClassName(keyType, payloadType);
this.className = this.generatedClassName.CleanUpIdentifierName();
this.generatedClassName += "`2";
this.expandedCode = TransformText();
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator only has to deal with start edges\r\n/// </summary>\r\n" +
"[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" [DataMember]\r\n private FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldStateStruct> heldAggregates;\r\n");
}
else
{
this.Write(" [DataMember]\r\n private HeldStateStruct currentState;\r\n [DataMember]\r\n " +
" private bool currentStateExists;\r\n [DataMember]\r\n private bool isDirty;\r" +
"\n");
}
this.Write("\r\n [DataMember]\r\n private long lastSyncTime = long.MinValue;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" +
"ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write("\r\n this.keyComparer = stream.Properties.KeyEqualityComparer;\r\n\r\n th" +
"is.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" var generator = this.keyComparer.CreateFastDictionary3Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldStateStruct>(1, this.keyComparer.GetEqualsExpr().Compile(), this.keyCompare" +
"r.GetGetHashCodeExpr().Compile(), stream.Properties.QueryContainer);\r\n th" +
"is.heldAggregates = generator.Invoke();\r\n");
}
else
{
this.Write(" isDirty = false;\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n\r\n protected override void DisposeSta" +
"te() => this.batch.Free();\r\n\r\n public override int CurrentlyBufferedOutputCou" +
"nt => this.batch.Count;\r\n\r\n public override int CurrentlyBufferedInputCount =" +
"> ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "0" : "this.heldAggregates.Count"));
this.Write(";\r\n\r\n public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n var count = generatedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@" var col_key = generatedBatch.key.col;
fixed (long* col_vsync = generatedBatch.vsync.col)
fixed (long* col_vother = generatedBatch.vother.col)
fixed (int* col_hash = generatedBatch.hash.col)
fixed (long* col_bv = generatedBatch.bitvector.col)
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == long.MinValue)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = long.MinValue;
this.batch.key.col[c] = default;
this.batch[c] = default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
");
if (this.isUngrouped)
{
this.Write(@" if (currentStateExists && isDirty) // there exists earlier state
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write("\r\n this.batch.hash.col[c] = 0;\r\n this.batch" +
".Count++;\r\n if (this.batch.Count == Config.DataBatchSize) Flu" +
"shContents();\r\n }\r\n isDirty = false;\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldStateStruct>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clean();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n " +
" this.lastSyncTime = syncTime;\r\n }\r\n\r\n");
if (this.isUngrouped)
{
this.Write(" if (!currentStateExists)\r\n {\r\n currentState" +
".state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
currentStateExists = true;
isDirty = true;
}
else
{
if (!isDirty)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write(@"
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
currentState.timestamp = syncTime;
isDirty = true;
}
}
currentState.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state", "col_vsync[i]")));
this.Write(";\r\n");
}
else
{
this.Write(@" // Retrieve the key from the dictionary
var currentKey = col_key[i];
var currentHash = col_hash[i];
int index;
bool heldAggsLookup = false;
{
int num = currentHash & 0x7fffffff;
index = num % this.heldAggregates.Size;
do
{
if ((this.heldAggregates.bitvector[index >> 3] & (0x1 << (index & 0x7))) == 0)
{
heldAggsLookup = false;
break;
}
if ((currentHash == this.heldAggregates.entries[index].hash) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "this.heldAggregates.entries[index].key")));
this.Write(@"))
{
heldAggsLookup = true;
break;
}
index++;
if (index == this.heldAggregates.Size)
index = 0;
} while (true);
}
if (!heldAggsLookup)
// if (!this.heldAggregates.Lookup(currentKey, currentHash, out index))
{
// New group. Create new state
HeldStateStruct currentState;
currentState.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
// No output because initial state is empty
this.heldAggregates.Insert(ref index, currentKey, currentState, currentHash);
}
else
{
// read new currentState from _heldAgg index
var currentState = this.heldAggregates.entries[index].value;
if (this.heldAggregates.IsClean(ref index))
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write(@"
this.batch.key.col[c] = currentKey;
this.batch.hash.col[c] = currentHash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
currentState.timestamp = syncTime;
this.heldAggregates.SetDirty(ref index);
}
}
this.heldAggregates.entries[index].value.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("this.heldAggregates.entries[index].value.state", "col_vsync[i]")));
this.Write(";\r\n");
}
this.Write(" }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n " +
" public void OnPunctuation(long syncTime)\r\n {\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Handle time moving forward\r\n if (syncTime > this.lastSyncTime" +
")\r\n {\r\n");
if (this.isUngrouped)
{
this.Write(@" if (currentStateExists && isDirty) // need to send start edge if state is dirty
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write("\r\n\r\n this.batch.hash.col[c] = 0;\r\n this.batch.Count" +
"++;\r\n if (this.batch.Count == Config.DataBatchSize) FlushContents" +
"();\r\n }\r\n isDirty = false;\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldStateStruct>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clean();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n this" +
".lastSyncTime = syncTime;\r\n }\r\n }\r\n}//\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Linq.Expressions;
using System.IO;
using System.Reflection;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n");
string TKey;
var genericTypeNames = new string[] { "A", "B" };
if (keyType.IsAnonymousType())
{
TKey = genericTypeNames[0];
}
else
{
genericTypeNames[0] = string.Empty;
TKey = keyType.GetCSharpSourceSyntax();
}
string TPayload;
var payloadIsAnon = false;
if (payloadType.IsAnonymousType())
{
TPayload = genericTypeNames[1];
payloadIsAnon = true;
}
else
{
genericTypeNames[1] = string.Empty;
TPayload = payloadType.GetCSharpSourceSyntax();
}
var genericParameters = Transformer.GenericParameterList(genericTypeNames);
var genericParameters2 = $"<{TKey}, {TPayload}>";
if (payloadType == typeof(int) || payloadType == typeof(long) || payloadType == typeof(string))
{
genericParameters2 = string.Empty;
}
this.Write("\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.payloadType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n");
if (payloadIsAnon)
{
foreach (var f in this.fields)
{
this.Write(" private PropertyInfo ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property = typeof(");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(").GetProperty(\"");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("\");\r\n");
}
}
this.Write("\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n this.pool = MemoryManager.GetMe" +
"moryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(">(true) as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.payloadType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(@";
this.queryPlanGenerator = queryPlanGenerator;
}
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(this.queryPlanGenerator(previous, this));
}
protected override void FlushContents() { }
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> batch)\r\n {\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> resultBatchUncast; // Need this type to call Get with so the right subtype will" +
" be returned\r\n this.pool.Get(out resultBatchUncast);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" resultBatch = resultBatchUncast as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(";\r\n\r\n\r\n resultBatch.CloneFromNoPayload(batch);\r\n\r\n // Get memory po" +
"ols for the payload columns.\r\n\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n this.pool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
if (this.payloadType.CanContainNull())
{
this.Write(" this.pool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write("\r\n var count = batch.Count;\r\n\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.Name));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n\r\n");
}
}
this.Write("\r\n fixed (long* bv = batch.bitvector.col)\r\n {\r\n");
if (this.payloadType.CanBeFixed())
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("* srcpayload = batch.payload.col)\r\n {\r\n");
}
else
{
this.Write("\r\n var srcpayload = batch.payload.col;\r\n\r\n");
}
this.Write("\r\n");
if (this.rowMajor)
{
this.Write(" for (int i = 0; i < count; i++)\r\n {\r\n if ((" +
"bv[i >> 6] & (1L << (i & 0x3f)))==0)\r\n {\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (payloadIsAnon)
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type));
this.Write(") this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property.GetValue(srcpayload[i], null);\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = srcpayload[i].");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
}
this.Write(" }\r\n }\r\n");
}
else
{
this.Write("\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n for (int i = 0; i < count; i++)\r\n {\r\n " +
" if ((bv[i >> 6] & (1L << (i & 0x3f)))==0)\r\n " +
" {\r\n");
if (payloadIsAnon)
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type));
this.Write(") this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_Property.GetValue(srcpayload[i], null);\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = srcpayload[i].");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
this.Write(" }\r\n }\r\n");
}
this.Write("\r\n");
}
this.Write("\r\n");
if (this.payloadType.CanBeFixed())
{
this.Write(" }\r\n");
}
this.Write(" }\r\n\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n");
}
this.Write(@"
resultBatch.Count = count;
resultBatch.Seal();
this.Observer.OnNext(resultBatch);
batch.Release();
batch.Return();
}
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
if (this.resultType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TPayload: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n// Source Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName))));
this.Write("\r\n// Destination Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.resultFields.Select(f => f.OriginalName))));
this.Write("\r\n// Computed Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName))));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" +
"rator;\r\n\r\n [DataMember]\r\n private int iter;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" batch;\r\n\r\n // Fields used to point directly to the arrays within the result b" +
"atch\r\n private long[] dest_vsync;\r\n private long[] dest_vother;\r\n priva" +
"te ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("[] destkey;\r\n private int[] dest_hash;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" +
"ool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(@";
this.queryPlanGenerator = queryPlanGenerator;
MyAllocate();
}
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
private void MyAllocate()
{
pool.Get(out genericBatch);
genericBatch.Allocate();
this.batch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@";
this.UpdatePointers();
}
protected override void UpdatePointers()
{
// Assign pointers to bookkeeping arrays
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
protected override void DisposeState() => this.batch.Free();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> _inBatch)\r\n {\r\n var batch = _inBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters));
this.Write(";\r\n\r\n var count = batch.Count;\r\n this.batch.iter = batch.iter;\r\n\r\n " +
" // Create locals that point directly to the arrays within the columns in " +
"the destination batch.\r\n\r\n ");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the source batch.\r\n ");
foreach (var f in this.fields)
{
this.Write("\r\n ");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n ");
}
else
{
this.Write("\r\n var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n var srckey = batch.key.col;\r\n ");
if (this.hasKey)
{
this.Write("\r\n var key_col = srckey; // hack until MakeColumnOriented is fixed\r\n " +
" ");
}
this.Write(@"
fixed (long* src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
fixed (int* src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
");
if (this.StartEdgeParameterName != null)
{
this.Write("\r\n var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName));
this.Write(" = src_vsync[i] < src_vother[i] ? src_vsync[i] : src_vother[i];\r\n " +
" ");
}
this.Write("\r\n ");
if (this.useEnumerator)
{
this.Write("\r\n var enumerator = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(".GetEnumerator();\r\n while (enumerator.MoveNext())\r\n " +
" ");
}
else
{
this.Write(" ");
if (!this.enumerableRepeatSelector)
{
this.Write("\r\n var e_prime = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(";\r\n ");
}
this.Write("\r\n ");
if (this.keyParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyParameterName));
this.Write(" = srckey[i];\r\n ");
}
this.Write(" for (int _x = 0; _x < ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.loopCounter));
this.Write("; _x++)\r\n ");
}
this.Write("\r\n {\r\n dest_vsync[iter] = src_vsync[i];" +
"\r\n dest_vother[iter] = src_vother[i];\r\n " +
" ");
if (this.useEnumerator)
{
this.Write("\r\n this.batch[iter] = enumerator.Current;\r\n " +
" ");
}
else if (this.enumerableRepeatSelector)
{
this.Write("\r\n ");
foreach (var kv in this.computedFields)
{
var f = kv.Key;
var v = kv.Value.ExpressionToCSharp();
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[iter] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n ");
}
this.Write(" ");
}
this.Write(" ");
}
else
{
this.Write("\r\n this.batch[iter] = e_prime;\r\n ");
}
this.Write(@"
destkey[iter] = srckey[i];
dest_hash[iter] = src_hash[i];
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n this.batch.iter = batch.iter;\r\n " +
" }\r\n }\r\n ");
if (this.useEnumerator)
{
this.Write("\r\n enumerator.Dispose();\r\n ");
}
this.Write(@" }
else if (src_vother[i] < 0)
{
dest_vsync[iter] = src_vsync[i];
dest_vother[iter] = src_vother[i];
destkey[iter] = default;
this.batch[iter] = default;
dest_hash[iter] = src_hash[i];
this.batch.bitvector.col[(iter) >> 6] |= (1L << ((iter) & 0x3f));
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n this.batch.iter = batch.iter;\r\n }\r\n " +
" }\r\n }\r\n\r\n } // end src_hash, src_bv, src_vsync," +
" src_vother\r\n\r\n ");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n ");
}
this.Write(@"
batch.Free();
}
protected override void FlushContents()
{
if (iter == 0) return;
this.batch.Count = iter;
this.batch.Seal();
this.Observer.OnNext(this.batch);
iter = 0;
MyAllocate();
}
public override int CurrentlyBufferedOutputCount => iter;
public override int CurrentlyBufferedInputCount => 0;
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
[assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")]
// OuterKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write("\r\n// InnerKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write("\r\n// InnerResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n");
var memoryPoolGenericParameters = string.Format("<{0}, {1}>", TOuterKey, TResult);
if (resultType.MemoryPoolHasGetMethodFor())
{
memoryPoolGenericParameters = string.Empty;
}
var memoryPoolClassName = Transformer.GetMemoryPoolClassName(this.outerKeyType, this.resultType) + memoryPoolGenericParameters;
var inputKey = ungroupingFromCompound ? "CompoundGroupKey<" + TOuterKey + ", " + TInnerKey + ">" : TInnerKey;
this.Write("\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(CLASSNAME));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" :\r\n Pipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">, IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(inputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write(">\r\n{\r\n private readonly ");
this.Write(this.ToStringHelper.ToStringWithCulture(memoryPoolClassName));
this.Write(" outPool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanG" +
"enerator;\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n private readonly ColumnBatch<Microsoft.StreamProcessing.Empty> unitColumn;\r" +
"\n private readonly ColumnBatch<int> unitHashColumn;\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" private readonly ColumnBatch<");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("> sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(CLASSNAME));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(CLASSNAME));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n outPool = MemoryManager.GetMemo" +
"ryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true) as ");
this.Write(this.ToStringHelper.ToStringWithCulture(memoryPoolClassName));
this.Write(";\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n outPool.GetKey(out unitColumn);\r\n outPool.Get(out unitHashColumn" +
");\r\n Array.Clear(unitHashColumn.col, 0, unitHashColumn.col.Length);\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" outPool.Get(out sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n Array.Clear(sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col, 0, sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col.Length);\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
protected override void DisposeState()
{
");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.Return();\r\n this.unitHashColumn.Return();\r\n");
}
foreach (var f in this.unassignedFields)
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
this.Write(" }\r\n\r\n public unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(inputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerResult));
this.Write("> batch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchGenericParameters));
this.Write(" inputBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(inputBatchGenericParameters));
this.Write(";\r\n\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> tmp; // Need this type to call Get with so the right subtype will be returned\r\n" +
" outPool.Get(out tmp);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(" resultBatch = tmp as ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(";\r\n\r\n");
foreach (var f in this.computedFields.Keys)
{
this.Write("\r\n outPool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit)
{
this.Write("\r\n tmp.hash = batch.hash.MakeWritable(outPool.intPool);\r\n");
}
this.Write(" var count = batch.Count;\r\n\r\n tmp.vsync = batch.vsync;\r\n tmp" +
".vother = batch.vother;\r\n tmp.bitvector = batch.bitvector;\r\n\r\n // " +
"Assign the swinging fields\r\n");
foreach (var tuple in this.swingingFields)
{
var destField = tuple.Item1.Name;
var sourceField = tuple.Item2.Name;
this.Write("\r\n resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(";\r\n");
}
this.Write("\r\n");
if (resultType.CanContainNull())
{
this.Write("\r\n outPool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write("\r\n fixed (long *srcbv = batch.bitvector.col) {\r\n");
if (!ungroupingToUnit)
{
this.Write(" fixed (int *desthash = tmp.hash.col) {\r\n");
}
this.Write("\r\n // Get pointers to the arrays for the inner result fields\r\n");
foreach (var f in this.innerResultRepresentation.AllFields)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
this.Write("\r\n // Get pointers to the arrays for the result fields\r\n");
foreach (var f in this.computedFields.Keys)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col) {\r\n");
}
else
{
if (!f.OptimizeString())
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
}
}
this.Write("\r\n");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.IncrementRefCount(1);\r\n tmp.key = unitColumn;\r\n " +
" this.unitHashColumn.IncrementRefCount(1);\r\n tmp.hash = unitHashColum" +
"n;\r\n");
}
this.Write("\r\n");
foreach (var f in this.unassignedFields)
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".IncrementRefCount(1);\r\n resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit || this.computedFields.Any())
{
if (!ungroupingToUnit)
{
this.Write(" var srckey = batch.key.col;\r\n var destkey = tmp.key.col;\r\n" +
"");
}
this.Write("\r\n for (int i = 0; i < count; i++)\r\n {\r\n if " +
"((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n " +
" // Need to add empty strings to keep multistring indexing consistent\r\n");
foreach (var kv in this.computedFields)
{
var field = kv.Key;
if (field.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(field.Name));
this.Write(".AddString(string.Empty);\r\n");
}
}
this.Write(" continue;\r\n }\r\n");
if (!ungroupingToUnit)
{
this.Write(" destkey[i] = srckey[i].outerGroup;\r\n desthash[i] =" +
" destkey[i].GetHashCode();\r\n");
}
foreach (var kv in this.computedFields)
{
var f = kv.Key;
string v;
var map = new Dictionary <System.Linq.Expressions.ParameterExpression, string>();
map.Add(this.keyParameter, "batch.key.col[i].InnerGroup");
v = kv.Value.ExpressionToCSharpStringWithParameterSubstitution(map);
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
}
this.Write(" }\r\n");
}
this.Write("\r\n");
foreach (var f in this.computedFields.Keys.Where(fld => fld.canBeFixed))
{
this.Write("\r\n } // end of fixed for ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit)
{
this.Write("\r\n } // end of fixed for desthash\r\n");
}
this.Write(" } // end of fixed for srcbv\r\n\r\n tmp.Count = count;\r\n\r\n\r\n ba" +
"tch.ReleaseKey();\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n batch.hash.Return();\r\n");
}
this.Write("\r\n #region Return source columns as necessary.\r\n // This is all fie" +
"lds from the input batch *except* for any swinging fields.\r\n");
foreach (var f in this.innerResultRepresentation.AllFields.Where(f => !this.swingingFields.Any(tup => tup.Item2.Name == f.Name)))
{
if (f.OptimizeString())
{
this.Write(" inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Dispose();\r\n");
}
else
{
this.Write(" inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
}
this.Write(" #endregion\r\n\r\n");
if (innerResultType.CanContainNull())
{
this.Write("\r\n inputBatch._nullnessvector.ReturnClear();\r\n");
}
this.Write("\r\n batch.Return();\r\n\r\n tmp.Seal();\r\n this.Observer.OnNext(tm" +
"p);\r\n }\r\n\r\n public void OnError(Exception error)\r\n {\r\n Observer." +
"OnError(error);\r\n }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator that uses a full-fledged priority queue as ECQ\r\n///" +
" </summary>\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n\r\n [DataContract]\r\n class EcqState\r\n {\r\n [DataMembe" +
"r]\r\n public long timestamp;\r\n [DataMember]\r\n public FastDic" +
"tionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive> states;\r\n }\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> deaccumulate;\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> difference;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool> keyComparerEquals;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", int> keyComparerGetHashCode;\r\n\r\n [DataMember]\r\n private FastDictionary2<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> aggregateByKey;\r\n [DataMember]\r\n private FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> heldAggregates;\r\n [DataMember]\r\n private SortedDictionary<long" +
", EcqState> ecq;\r\n [DataMember]\r\n private long lastSyncTime = long.MinValu" +
"e;\r\n\r\n private HeldState currentState;\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentKey;\r\n private int currentHash;\r\n\r\n private Func<FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>> stateDictGenerator;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" +
"ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" deaccumulate = aggregate.Deaccumulate().Compile();\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" difference = aggregate.Difference().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write(@"
this.keyComparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEquals = this.keyComparer.GetEqualsExpr().Compile();
this.keyComparerGetHashCode = this.keyComparer.GetGetHashCodeExpr().Compile();
this.stateDictGenerator = this.keyComparer.CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream." +
"Properties.QueryContainer);\r\n\r\n this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n this.aggregateByKey = this.keyCompare" +
"r.CreateFastDictionary2Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Prope" +
"rties.QueryContainer).Invoke();\r\n this.heldAggregates = this.keyComparer." +
"CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke();
this.ecq = new SortedDictionary<long, EcqState>();
}
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
this.batch.Allocate();
}
protected override void DisposeState() => this.batch.Free();
public override int CurrentlyBufferedOutputCount => this.batch.Count;
public override int CurrentlyBufferedInputCount => this.aggregateByKey.Count + this.ecq.Values.Select(o => o.states.Count).Sum();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n this.batch.iter = generatedBatch.iter;\r\n\r\n var count = genera" +
"tedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n fixed (long* col_vsync = generatedBatch.vsync.col)\r\n fixed (long" +
"* col_vother = generatedBatch.vother.col)\r\n fixed (int* col_hash = genera" +
"tedBatch.hash.col)\r\n fixed (long* col_bv = generatedBatch.bitvector.col)\r" +
"\n for (int i = 0; i < count; i++)\r\n {\r\n if ((col_bv[i >" +
"> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n if (col_vother[" +
"i] == long.MinValue)\r\n {\r\n // We have found a " +
"row that corresponds to punctuation\r\n OnPunctuation(col_vsync" +
"[i]);\r\n\r\n int c = this.batch.Count;\r\n this" +
".batch.vsync.col[c] = col_vsync[i];\r\n this.batch.vother.col[c" +
"] = long.MinValue;\r\n this.batch.key.col[c] = default;\r\n " +
" this.batch[c] = default;\r\n this.batch.hash.col[" +
"c] = 0;\r\n this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f" +
");\r\n this.batch.Count++;\r\n if (this.batch." +
"Count == Config.DataBatchSize) FlushContents();\r\n }\r\n " +
" continue;\r\n }\r\n\r\n var syncTime = col_vsync[i];\r\n " +
" var key_i = generatedBatch.key.col[i];\r\n\r\n HeldState heldState;" +
"\r\n bool cachedState = false;\r\n\r\n // Handle time moving for" +
"ward\r\n if (syncTime > this.lastSyncTime)\r\n {\r\n " +
" /* Issue start edges for held aggregates */\r\n if (currentStat" +
"e != null && this.heldAggregates.Count == 1)\r\n {\r\n " +
" // there is just one held aggregate, and currentState is set\r\n " +
" // so currentState has to be the held aggregate\r\n cach" +
"edState = true;\r\n\r\n if (currentState.state.active > 0)\r\n " +
" {\r\n int c = this.batch.Count;\r\n " +
" this.batch.vsync.col[c] = currentState.timestamp;\r\n " +
" this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;\r\n " +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.iter = generatedBatch.iter;
FlushContents();
this.batch.iter = generatedBatch.iter;
}
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash);
currentState = null;
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
}
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))" +
"\r\n {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Update aggregate
heldState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries" +
"[iter].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n " +
" if (this.batch.Count == Config.DataBatchSize) FlushContents();\r\n " +
" }\r\n else\r\n " +
" {\r\n // remove from aggregateByKey\r\n " +
" this.aggregateByKey.Remove(ecqState.states.entries[iter].k" +
"ey);\r\n currentState = null;\r\n " +
" }\r\n }\r\n else\r\n " +
" {\r\n if (cachedState)\r\n " +
" {\r\n cachedState = false;\r\n " +
" if (heldState != currentState)\r\n " +
" {\r\n this.heldAggregates.Clear();\r\n " +
" currentState = null;\r\n " +
" int index;\r\n this.heldAggregates.Lookup(" +
"ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n " +
" else\r\n {\r\n " +
" int index;\r\n this.heldAggregates.Loo" +
"kup(ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n\r\n " +
" // Update timestamp\r\n heldState.timestamp =" +
" ecqState.timestamp;\r\n }\r\n }\r\n\r\n " +
" // Since sync time changed, set this.lastSyncTime\r\n this.lastSy" +
"ncTime = syncTime;\r\n }\r\n\r\n if (currentState == null || cur" +
"rentHash != col_hash[i] || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "key_i")));
this.Write(@")
{
if (cachedState)
{
cachedState = false;
this.heldAggregates.Clear();
}
// Need to retrieve the key from the dictionary
currentKey = key_i;
currentHash = col_hash[i];
int index;
if (!this.heldAggregates.Lookup(currentKey, currentHash, out index))
{
// First time group is active for this time
int aggindex;
if (!this.aggregateByKey.Lookup(currentKey, currentHash, out aggindex))
{
// New group. Create new state
currentState = new HeldState();
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
this.aggregateByKey.Insert(currentKey, currentState, currentHash);
// No output because initial state is empty
}
else
{
currentState = this.aggregateByKey.entries[aggindex].value;
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
this.heldAggregates.Insert(ref index, currentKey, currentState);
}
else
{
// read new currentState from _heldAgg index
currentState = this.heldAggregates.entries[index].value;
}
}
else
{
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
if (col_vsync[i] < col_vother[i]) // insert event
{
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(@";
currentState.state.active++;
// Update ECQ
if (col_vother[i] < StreamEvent.InfinitySyncTime)
{
EcqState eState;
int index;
if (this.ecq.Count > 0)
{
if (!this.ecq.TryGetValue(col_vother[i], out eState))
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
this.ecq.Add(col_vother[i], eState);
}
else
{
if (!eState.states.Lookup(key_i, col_hash[i], out index))
{
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
}
}
}
else
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(" });\r\n this.ecq.Add(col_vother[i], eState);\r\n " +
" }\r\n\r\n eState.states.entries[index].value.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("eState.states.entries[index].value.state", "col_vsync[i]")));
this.Write(";\r\n eState.states.entries[index].value.active++;\r\n " +
" }\r\n }\r\n else // is a retraction\r\n {\r\n " +
" currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(deaccumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(";\r\n currentState.state.active--;\r\n }\r\n }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n\r" +
"\n public void OnPunctuation(long syncTime)\r\n {\r\n HeldState heldStat" +
"e;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@"
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
/* Issue start edges for held aggregates */
if (currentState != null && this.heldAggregates.Count == 1)
{
// there is just one held aggregate, and currentState is set
// so currentState has to be the held aggregate
if (currentState.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n th" +
"is.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key);
currentState = null;
}
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))\r\n " +
" {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[iter].k" +
"ey;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n if (this." +
"batch.Count == Config.DataBatchSize) FlushContents();\r\n }\r\n\r\n" +
" // Update aggregate\r\n heldState.state.sta" +
"te = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
// remove from aggregateByKey
this.aggregateByKey.Remove(ecqState.states.entries[iter].key);
currentState = null;
}
}
else
{
int index;
this.heldAggregates.Lookup(ecqState.states.entries[iter].key, out index);
this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, heldState);
}
// Update timestamp
heldState.timestamp = ecqState.timestamp;
}
}
// Since sync time changed, set this.lastSyncTime
this.lastSyncTime = syncTime;
}
}
protected override void UpdatePointers()
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>.IteratorStart;\r\n while (this.heldAggregates.Iterate(ref iter1" +
"))\r\n {\r\n int iter2;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = this.heldAggregates.entries[iter1].key;
bool found = this.aggregateByKey.Lookup(key, out iter2);
if (!found) throw new InvalidOperationException();
this.heldAggregates.entries[iter1].value = this.aggregateByKey.entries[iter2].value;
}
}
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null && this.payloadType.Namespace != this.keyType.Namespace)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
if (this.resultType.Namespace != null && this.resultType.Namespace != this.keyType.Namespace && this.resultType.Namespace != this.payloadType.Namespace)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace));
this.Write(";\r\n");
}
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// Source Fiel" +
"ds: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName))));
this.Write("\r\n// Destination Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.destinationFields.Select(f => f.OriginalName))));
this.Write("\r\n// Computed Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName))));
this.Write("\r\n// Swinging Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.swingingFields.Select(tup => string.Format("<{0},{1}>", tup.Item1.OriginalName, tup.Item2.Name)))));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" +
"rator;\r\n");
foreach (var f in this.unassignedFields)
{
if (!f.OptimizeString())
{
this.Write("\r\n private readonly ColumnBatch<");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("> sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" +
"ool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(";\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString()))
{
this.Write("\r\n pool.Get(out sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n Array.Clear(sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col, 0, sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col.Length);\r\n");
}
this.Write("\r\n }\r\n\r\n public override void ProduceQueryPlan(PlanNode previous)\r\n {\r\n " +
" Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));\r\n }\r\n\r\n" +
" protected override void DisposeState()\r\n {\r\n");
foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString()))
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
this.Write(" }\r\n\r\n public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> _inBatch)\r\n {\r\n var sourceBatch = _inBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters));
this.Write(";\r\n\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> _genBatch; // Need this type to call Get with so the right subtype will be retu" +
"rned\r\n pool.Get(out _genBatch);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" resultBatch = _genBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@";
var count = sourceBatch.Count;
resultBatch.vsync = sourceBatch.vsync;
resultBatch.vother = sourceBatch.vother;
resultBatch.key = sourceBatch.key;
resultBatch.hash = sourceBatch.hash;
resultBatch.bitvector = sourceBatch.bitvector;
");
if (resultType.CanContainNull())
{
this.Write(" pool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write(@"
// Get memory pools for the result columns.
// When no transformation was done to the query, then needed for all fields in the result type.
// When the query was transformed, then this is needed only for computed fields, since
// any field that is just assigned a field from the source type will get its value by
// swinging a pointer for the corresponding column.
");
foreach (var f in (this.ProjectionReturningResultInstance != null ? this.destinationFields : this.computedFields.Keys))
{
this.Write(" pool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
this.Write("\r\n");
foreach (var f in this.unassignedFields)
{
if (!f.OptimizeString())
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".IncrementRefCount(1);\r\n");
}
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the destination batch.\r\n\r\n");
foreach (var f in this.computedFields.Keys)
{
if (f.OptimizeString())
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the source batch.\r\n");
if (this.ProjectionReturningResultInstance != null || 0 < this.computedFields.Count())
{
foreach (var f in this.fields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n fixed (long* bv = sourceBatch.bitvector.col)\r\n {\r\n fo" +
"r (int i = 0; i < count; i++)\r\n {\r\n if ((bv[i >> 6] & " +
"(1L << (i & 0x3f))) == 0)\r\n {\r\n");
if (this.StartEdgeParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName));
this.Write(" = sourceBatch.vsync.col[i] < sourceBatch.vother.col[i] ? sourceBatch.vsync.col[i" +
"] : sourceBatch.vother.col[i];\r\n");
}
if (this.needSourceInstance)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(PARAMETER));
this.Write(" = sourceBatch[i];\r\n");
}
if (this.ProjectionReturningResultInstance != null)
{
this.Write(" resultBatch[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.ProjectionReturningResultInstance));
this.Write(";\r\n");
}
else
{
foreach (var kv in this.computedFields)
{
var f = kv.Key;
var v = kv.Value.ExpressionToCSharp();
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
}
}
this.Write(" }\r\n }\r\n }\r\n\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
}
this.Write("\r\n #region MultiString Vector Operations (if any)\r\n");
foreach (var mso in this.multiStringOperations)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(mso));
this.Write("\r\n");
}
this.Write(@" #endregion
#region Swinging Fields
// When the query is transformed, then any fields in the result batch that
// are just assigned from a field in the source batch are computed by just
// swinging the pointer for the corresponding column.
");
foreach (var tuple in this.swingingFields)
{
var destField = tuple.Item1.Name;
var sourceField = tuple.Item2.Name;
this.Write("\r\n // Special case for a field in the output being assigned the key field\r" +
"\n // when both are strings and the output field is a MultiString. The key" +
"\r\n // field is a CB<string> so we need to call MultiString.FromColumnBatc" +
"h\r\n");
if (tuple.Item1.OptimizeString() && sourceField.Equals("key"))
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = MultiString.FromColumnBatch(sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(", sourceBatch.bitvector, pool.charArrayPool, pool.intPool, pool.shortPool, pool.b" +
"itvectorPool);\r\n");
}
else
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(";\r\n");
}
}
this.Write(@"
// We avoid incrementing the ref counts on the ouput batch because for the
// most part we are throwing away our reference to the columns from the source batch
// so the reference count doesn't change. But the same field might get swung to more
// than one column in the output batch. In that case, we need to increment the ref count.
");
foreach (var kv in this.swungFieldsCount)
{
var sourceField = kv.Key;
var count = kv.Value;
if (sourceField.Name.Equals("key") && !sourceField.OptimizeString())
{
// special case: we "pointer swing" the key field once just because the key
// of the output batch is the key of the source batch
// (Note that this really applies only to SelectKey since Select is not able
// to reference the key field in the selector.)
count++;
}
if (count == 1)
{
continue;
}
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".IncrementRefCount(");
this.Write(this.ToStringHelper.ToStringWithCulture(count - 1));
this.Write(");\r\n");
}
this.Write(@" #endregion
resultBatch.Count = count;
resultBatch.Seal();
// Return source columns as necessary.
// When the query is transformed, this is the ""non-swinging"" fields.
// Otherwise it is all source fields
");
foreach (var sourceField in this.nonSwingingFields)
{
if (sourceField.OptimizeString())
{
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".Dispose();\r\n");
}
else
{
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".Return();\r\n");
}
}
if (payloadType.CanContainNull())
{
this.Write(" sourceBatch._nullnessvector.ReturnClear();\r\n");
}
this.Write("\r\n sourceBatch.Return();\r\n this.Observer.OnNext(resultBatch);\r\n " +
"}\r\n\r\n public override int CurrentlyBufferedOutputCount => 0;\r\n\r\n public ov" +
"erride int CurrentlyBufferedInputCount => 0;\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
