public SnapshotWindowStreamable(IStreamable <TKey, TInput> source, IAggregate <TInput, TState, TOutput> aggregate)
: base(source, source.Properties.Snapshot(aggregate))
{
Contract.Requires(source != null);
this.Aggregate = aggregate;
this.sourceProps = source.Properties;
if (this.sourceProps.IsStartEdgeOnly)
{
this.apt = AggregatePipeType.StartEdge;
}
else if (this.sourceProps.IsConstantDuration)
{
if (this.sourceProps.IsTumbling)
{
this.apt = AggregatePipeType.Tumbling;
}
else if (this.sourceProps.ConstantDurationLength.HasValue && this.sourceProps.IsConstantHop && this.sourceProps.ConstantHopLength.HasValue)
{
this.apt = AggregatePipeType.Hopping;
}
else
{
this.apt = AggregatePipeType.Sliding;
}
}
else
{
this.apt = AggregatePipeType.PriorityQueue;
}
Initialize();
}
/// <summary>
/// Ungroup with group-selector
/// </summary>
internal static StreamProperties <TOuterKey, TResult> Ungroup <TOuterKey, TInnerKey, TPayload, TResult>(
this StreamProperties <CompoundGroupKey <TOuterKey, TInnerKey>, TPayload> source,
Expression <Func <TInnerKey, TPayload, TResult> > groupSelector)
{
IEqualityComparerExpression <TOuterKey> newKeyEqualityComparer = null;
if (source.KeyEqualityComparer != null)
{
newKeyEqualityComparer = ((CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>)source.KeyEqualityComparer).outerComparer;
}
IComparerExpression <TOuterKey> newKeyComparer = null;
if (source.KeyComparer != null)
{
newKeyComparer = ((CompoundGroupKeyComparer <TOuterKey, TInnerKey>)source.KeyComparer).outerComparer;
}
return(new StreamProperties <TOuterKey, TResult>(
source.IsColumnar, source.IsConstantDuration, source.ConstantDurationLength,
source.IsConstantHop, source.ConstantHopLength, source.ConstantHopOffset,
source.IsIntervalFree, false,
false, false,
newKeyEqualityComparer,
EqualityComparerExpression <TResult> .Default,
newKeyComparer,
null,
new Dictionary <Expression, object>(),
new Dictionary <Expression, Guid?>(),
source.QueryContainer));
}
void UpdatingForExistingStream()
{
var partition = new Partition(Table, Id + ".c");
var properties = new Dictionary <string, EntityProperty>
{
{ "Created", new EntityProperty(DateTimeOffset.Now) },
{ "Active", new EntityProperty(true) }
};
Stream.Provision(partition, StreamProperties.From(properties));
Console.WriteLine("Stream metadata specified for stream in partition '{0}'",
partition);
var stream = Stream.Open(partition);
Print(stream.Properties);
properties["Active"] = new EntityProperty(false);
Stream.SetProperties(stream, StreamProperties.From(properties));
Console.WriteLine("Updated stream metadata in partition '{0}'", partition);
stream = Stream.Open(partition);
Print(stream.Properties);
}
public async void When_set_successfully()
{
var properties = new Dictionary <string, EntityProperty>
{
{ "P1", new EntityProperty(42) },
{ "P2", new EntityProperty("42") }
};
var stream = await Stream.ProvisionAsync(partition, StreamProperties.From(properties));
var newProperties = new Dictionary <string, EntityProperty>
{
{ "P1", new EntityProperty(56) },
{ "P2", new EntityProperty("56") }
};
var newStream = await Stream.SetPropertiesAsync(stream, StreamProperties.From(newProperties));
StreamProperties.From(newProperties).ToExpectedObject().ShouldEqual(newStream.Properties);
var storedEntity = partition.RetrieveStreamEntity();
var storedProperties = storedEntity.Properties;
StreamProperties.From(newProperties).ToExpectedObject().ShouldEqual(storedProperties);
}
public async Task When_writing_to_existing_stream_non_null_stream_properties()
{
var properties = new
{
Property1 = "Foo",
Property2 = 42
};
var stream = new Stream(partition, StreamProperties.From(properties));
var result = await Stream.WriteAsync(stream, CreateEvent("e1"), CreateEvent("e2"));
stream = result.Stream;
AssertNewStream(result, 2, properties);
AssertStreamEntity(2, properties);
var replaced = new
{
Property1 = "Bar"
};
var restored = Stream.From(partition, stream.ETag, stream.Version, StreamProperties.From(replaced));
result = await Stream.WriteAsync(restored, CreateEvent("e3"));
AssertNewStream(result, 3, replaced);
AssertStreamEntity(3, replaced);
}
private void TestStreamPropertiesClone(StreamProperties sp)
{
// Set members in source object
sp.ContentType = "foo_1";
sp.ContentEncoding = "foo_2";
sp.Headers = new Dictionary <string, object>();
sp.Headers.Add("foo_3", "foo_4");
sp.Headers.Add("foo_5", "foo_6");
sp.Priority = 12;
sp.Timestamp = new AmqpTimestamp(123);
// Clone
StreamProperties spClone = sp.Clone() as StreamProperties;
// Change values in source object
sp.ContentType = "foo_7";
sp.ContentEncoding = "foo_8";
sp.Headers.Remove("foo_3");
sp.Headers.Remove("foo_5");
sp.Headers.Add("foo_9", "foo_10");
sp.Headers.Add("foo_11", "foo_12");
sp.Priority = 34;
sp.Timestamp = new AmqpTimestamp(234);
// Make sure values have not changed in clone
Assert.AreEqual("foo_1", spClone.ContentType);
Assert.AreEqual("foo_2", spClone.ContentEncoding);
Assert.AreEqual(2, spClone.Headers.Count);
Assert.AreEqual(true, spClone.Headers.ContainsKey("foo_3"));
Assert.AreEqual("foo_4", spClone.Headers["foo_3"]);
Assert.AreEqual(true, spClone.Headers.ContainsKey("foo_5"));
Assert.AreEqual("foo_6", spClone.Headers["foo_5"]);
Assert.AreEqual(12, spClone.Priority);
Assert.AreEqual(new AmqpTimestamp(123), spClone.Timestamp);
}
private StreamProperties GetPropertiesFromStream(Stream stream, bool isImage)
{
StreamProperties streamProperties = new StreamProperties {
FileSize = 0, ImageHeight = 0, ImageWidth = 0
};
using (MemoryStream memoryStream = new MemoryStream())
{
stream.CopyTo(memoryStream);
memoryStream.Position = 0;
streamProperties.FileSize = memoryStream.Length;
if (isImage)
{
try
{
using (Image image = Image.FromStream(memoryStream))
{
streamProperties.ImageWidth = image.Width;
streamProperties.ImageHeight = image.Height;
}
}
catch (Exception ex)
{
EventLog.Application.WriteWarning("Failed to get image from stream. Image entity width and height will not be set. Exception {0}.", ex.ToString());
streamProperties.ImageWidth = 0;
streamProperties.ImageHeight = 0;
}
}
return(streamProperties);
}
}
public ShardedCacheObserver(StreamCache <TKey, TPayload> cache, StreamProperties <TKey, TPayload> sourceProps)
{
this.cache = cache;
this.sourceProps = sourceProps;
this.elements = new List <QueuedMessage <StreamMessage <TKey, TPayload> > >();
}
public async Task When_provisioning_along_with_custom_properties()
{
var properties = new Dictionary <string, EntityProperty>
{
{ "Created", new EntityProperty(DateTimeOffset.Now) },
{ "Active", new EntityProperty(true) }
};
var stream = await Stream.ProvisionAsync(partition, StreamProperties.From(properties));
var entity = partition.RetrieveStreamEntity();
var expectedStream = new Stream
(
partition,
entity.ETag, 0,
StreamProperties.From(properties)
);
expectedStream.ToExpectedObject().ShouldEqual(stream);
var expectedEntity = new
{
RowKey = Api.StreamRowKey,
Properties = StreamProperties.From(properties),
Version = 0
};
expectedEntity.ToExpectedObject().ShouldMatch(entity);
}
Stream CreateStream(int version, string etag, object properties = null)
{
var props = properties != null
? StreamProperties.From(properties)
: StreamProperties.None;
return(new Stream(partition, etag, version, props));
}
protected ConnectableStreamableBase(StreamProperties <TKey, TPayload> properties)
: base(properties)
{
Contract.Requires(properties != null);
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(properties.IsColumnar);
this.ClassId = Guid.NewGuid();
}
public BinaryStreamObserver(StreamProperties <TKey, TPayload> streamProperties, Stream stream)
{
this.serializer = StreamSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(
new SerializerSettings()
{
KnownTypes = StreamMessageManager.GeneratedTypes()
});
this.stream = stream;
}
public void When_passing_property_with_reserved_name()
{
var reserved = ReservedStreamProperties()
.ToDictionary(p => p, p => new EntityProperty(42));
var properties = StreamProperties.From(reserved);
Assert.That(properties.Count, Is.EqualTo(0),
"Should skip all properties with reserved names, such as RowKey, Id, etc");
}
Map2Reduce <TMapInputLeft1, TMapInputRight1, TMapInputLeft2, TMapInputRight2, TReduceKey, TReduceInput1, TReduceInput2, TBind, TOutput>(
this StreamProperties <Empty, TMapInputLeft1> leftStream1,
StreamProperties <Empty, TMapInputRight1> rightStream1,
StreamProperties <Empty, TMapInputLeft2> leftStream2,
StreamProperties <Empty, TMapInputRight2> rightStream2,
Func <IStreamable <Empty, TMapInputLeft1>, IStreamable <Empty, TMapInputRight1>, IStreamable <Empty, TReduceInput1> > mapper1,
Func <IStreamable <Empty, TMapInputLeft2>, IStreamable <Empty, TMapInputRight2>, IStreamable <Empty, TReduceInput2> > mapper2,
Func <IStreamable <TReduceKey, TReduceInput1>,
IStreamable <TReduceKey, TReduceInput2>,
IStreamable <TReduceKey, TBind> > reducer,
Expression <Func <TReduceInput1, TReduceKey> > keySelector1,
Expression <Func <TReduceInput2, TReduceKey> > keySelector2,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector,
OperationalHint reduceOptions)
{
var map1 = leftStream1.Derive(a => mapper1(a, null));
var map2 = leftStream2.Derive(a => mapper2(a, null));
var group1 = map1.Group(keySelector1);
var group2 = map2.Group(keySelector2);
if (leftStream1.CanSpray(leftStream2, keySelector1, keySelector2) &&
group1.CanSpray(group2, keySelector1, keySelector2))
{
var reduce = group1.Derive(group2, reducer).Ungroup(resultSelector);
if (Config.MapArity > 1)
{
reduce = reduce.Union(reduce);
}
return(reduce);
}
else
{
if (Config.MapArity > 1)
{
group1 = group1.Union(group1);
group2 = group2.Union(group2);
}
var reduce = group1.Derive(group2, reducer).Ungroup(resultSelector);
if (Config.ReduceArity > 1)
{
reduce = reduce.Union(reduce);
}
return(reduce);
}
}
protected BinaryStreamable(StreamProperties <TKey, TResult> properties, IStreamable <TKey, TLeft> left,
IStreamable <TKey, TRight> right, bool registerInputs = false)
: base(properties)
{
Contract.Requires(left != null);
Contract.Requires(right != null);
this.Left = left;
this.Right = right;
this.registerInputs = registerInputs;
}
public async void When_property_map_is_empty()
{
var properties = new Dictionary <string, EntityProperty>();
var previous = await Stream.ProvisionAsync(partition);
var current = await Stream.SetPropertiesAsync(previous, StreamProperties.From(properties));
Assert.That(current.ETag, Is.Not.EqualTo(previous.ETag));
StreamProperties.From(properties).ToExpectedObject().ShouldEqual(current.Properties);
}
/// <summary>
/// Map + single-input Reduce
/// </summary>
internal static StreamProperties <TKey, TOutput> MapReduce <TKey, TMapInputLeft, TMapInputRight, TReduceInput, TReduceKey, TBind, TOutput>(
this StreamProperties <TKey, TMapInputLeft> leftStream,
StreamProperties <TKey, TMapInputRight> rightStream,
Func <IStreamable <TKey, TMapInputLeft>, IStreamable <TKey, TMapInputRight>, IStreamable <TKey, TReduceInput> > mapper,
Expression <Func <TReduceInput, TReduceKey> > keySelector,
Func <IStreamable <CompoundGroupKey <TKey, TReduceKey>, TReduceInput>, IStreamable <CompoundGroupKey <TKey, TReduceKey>, TBind> > reducer,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector)
{
bool mc = leftStream.IsMulticore;
var map = leftStream.ToMulticore(true).Derive(a => mapper(a, null));
if (reducer != null)
{
// We need to test the reducer function to see what it does to properties.
var group = map.GroupNested(keySelector);
if (group.CanSpray(keySelector))
{
var reduce = group.Derive(reducer).Ungroup(resultSelector);
if (Config.MapArity > 1)
{
reduce = reduce.Union(reduce);
}
var returnValue = reduce.ToMulticore(mc);
return(returnValue);
}
else
{
if (Config.MapArity > 1)
{
group = group.Union(group);
}
var reduce = group.Derive(reducer).Ungroup(resultSelector);
if (Config.ReduceArity > 1)
{
reduce = reduce.Union(reduce);
}
var returnValue = reduce.ToMulticore(mc);
return(returnValue);
}
}
else
{
var result = leftStream.Derive(a => mapper(a, null)) as StreamProperties <TKey, TOutput>;
if (Config.MapArity > 1)
{
result = result.Union(result);
}
return(result.ToMulticore(mc));
}
}
// TODO: This appears to be copied code from Binary egress - can we unify?
public ShardedSerializerObserver(Stream destination, StreamProperties <TKey, TPayload> sourceProps, bool writePropertiesToStream = false)
{
this.destination = destination;
if (writePropertiesToStream)
{
var propSer = StreamableSerializer.Create <SerializedProperties>();
propSer.Serialize(destination, SerializedProperties.FromStreamProperties(sourceProps));
}
this.serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(new SerializerSettings());
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TPayload"></typeparam>
/// <param name="fileName"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <returns></returns>
public static IPassiveIngressStreamable <TKey, TPayload> ToStreamablePassiveFromFile <TKey, TPayload>(
this string fileName,
bool readPropertiesFromStream = false,
StreamProperties <TKey, TPayload> inputProperties = null)
{
return(new BinaryIngressStreamablePassive <TKey, TPayload>(
new FileStream(fileName, FileMode.Open),
inputProperties,
readPropertiesFromStream,
null,
Guid.NewGuid().ToString()));
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TPayload"></typeparam>
/// <param name="binaryStream"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <returns></returns>
public static IPassiveIngressStreamable <TKey, TPayload> ToStreamablePassive <TKey, TPayload>(
this Stream binaryStream,
bool readPropertiesFromStream = false,
StreamProperties <TKey, TPayload> inputProperties = null)
{
return(new BinaryIngressStreamablePassive <TKey, TPayload>(
binaryStream,
inputProperties,
readPropertiesFromStream,
null,
Guid.NewGuid().ToString()));
}
public static SerializedProperties FromStreamProperties <TKey, TPayload>(StreamProperties <TKey, TPayload> props)
=> new SerializedProperties
{
IsColumnar = props.IsColumnar,
IsConstantDuration = props.IsConstantDuration,
ConstantDurationLength = props.ConstantDurationLength,
IsConstantHop = props.IsConstantHop,
ConstantHopLength = props.ConstantHopLength,
ConstantHopOffset = props.ConstantHopOffset,
IsIntervalFree = props.IsIntervalFree,
IsSyncTimeSimultaneityFree = props.IsSyncTimeSimultaneityFree,
IsEventOverlappingFree = props.IsEventOverlappingFree
};
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TPayload"></typeparam>
/// <param name="fileName"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <param name="container"></param>
/// <param name="identifier"></param>
/// <returns></returns>
public static IPassiveIngressStreamable <TKey, TPayload> RegisterBinaryInputPassiveFromFile <TKey, TPayload>(
this QueryContainer container,
string fileName,
bool readPropertiesFromStream = false,
StreamProperties <TKey, TPayload> inputProperties = null,
string identifier = null)
{
return(new BinaryIngressStreamablePassive <TKey, TPayload>(
new FileStream(fileName, FileMode.Open),
inputProperties,
readPropertiesFromStream,
container,
identifier ?? Guid.NewGuid().ToString()));
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TPayload"></typeparam>
/// <param name="binaryStream"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <param name="container"></param>
/// <param name="identifier"></param>
/// <returns></returns>
public static IPassiveIngressStreamable <Empty, TPayload> RegisterBinaryInputPassive <TPayload>(
this QueryContainer container,
Stream binaryStream,
bool readPropertiesFromStream = false,
StreamProperties <Empty, TPayload> inputProperties = null,
string identifier = null)
{
return(new BinaryIngressStreamablePassive <Empty, TPayload>(
binaryStream,
inputProperties,
readPropertiesFromStream,
container,
identifier ?? Guid.NewGuid().ToString()));
}
public StreamManagerTest()
{
_mockEventSource = new Mock <IEventSource>();
_mockEventSource.Setup(es => es.StartAsync()).Returns(Task.CompletedTask);
_eventSource = _mockEventSource.Object;
_eventSourceCreator = new StubEventSourceCreator(_eventSource);
_config = new SimpleConfiguration
{
SdkKey = SdkKey
};
_mockStreamProcessor = new Mock <IStreamProcessor>();
_streamProcessor = _mockStreamProcessor.Object;
_streamProperties = new StreamProperties(StreamUri, HttpMethod.Get, null);
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TPayload"></typeparam>
/// <param name="binaryStream"></param>
/// <param name="scheduler"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <returns></returns>
public static IIngressStreamable <Empty, TPayload> ToStreamable <TPayload>(
this Stream binaryStream,
IIngressScheduler scheduler = null,
bool readPropertiesFromStream = false,
StreamProperties <Empty, TPayload> inputProperties = null)
{
return(new BinaryIngressStreamable <Empty, TPayload>(
binaryStream,
0,
scheduler,
inputProperties,
readPropertiesFromStream,
null,
Guid.NewGuid().ToString()));
}
void AssertStreamEntity(int version = 0, object properties = null)
{
var newStreamEntity = partition.RetrieveStreamEntity();
var expectedEntity = new
{
RowKey = Api.StreamRowKey,
Properties = properties != null
? StreamProperties.From(properties)
: StreamProperties.None,
Version = version,
};
newStreamEntity.ShouldMatch(expectedEntity.ToExpectedObject());
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TPayload"></typeparam>
/// <param name="fileName"></param>
/// <param name="numMessages"></param>
/// <param name="scheduler"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <returns></returns>
public static IIngressStreamable <Empty, TPayload> ToStreamableFromFile <TPayload>(
this string fileName,
int numMessages = 0,
IIngressScheduler scheduler = null,
bool readPropertiesFromStream = false,
StreamProperties <Empty, TPayload> inputProperties = null)
{
return(new BinaryIngressStreamable <Empty, TPayload>(
new FileStream(fileName, FileMode.Open),
numMessages,
scheduler,
inputProperties,
readPropertiesFromStream,
null,
Guid.NewGuid().ToString()));
}
void SimultaneousSettingOfStreamMetadata()
{
var a = Stream.Open(Partition);
var b = Stream.Open(Partition);
Stream.SetProperties(a, StreamProperties.From(new { A = 42 }));
try
{
Stream.SetProperties(b, StreamProperties.From(new { A = 56 }));
}
catch (ConcurrencyConflictException)
{
Console.WriteLine("Simultaneously setting metadata using the same version of stream will lead to ConcurrencyConflictException");
}
}
public GroupedWindowStreamable(IStreamable <Empty, TInput> source, IAggregate <TInput, TState, TOutput> aggregate, Expression <Func <TInput, TKey> > keySelector, Expression <Func <TKey, TOutput, TResult> > resultSelector)
: base(source, source.Properties.Group(keySelector).Snapshot(aggregate).Ungroup(resultSelector))
{
Contract.Requires(source != null);
this.Aggregate = aggregate;
this.sourceProps = source.Properties;
this.KeySelector = keySelector;
this.ResultSelector = resultSelector;
if (!this.sourceProps.IsStartEdgeOnly)
{
throw new InvalidOperationException("Cannot use this streamable if the input stream is not guaranteed to be start-edge only.");
}
Initialize();
}
/// <summary>
/// Deserialize from binary stream to streamable
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TPayload"></typeparam>
/// <param name="binaryStream"></param>
/// <param name="scheduler"></param>
/// <param name="readPropertiesFromStream"></param>
/// <param name="inputProperties"></param>
/// <param name="container"></param>
/// <param name="identifier"></param>
/// <returns></returns>
public static IIngressStreamable <TKey, TPayload> RegisterBinaryInput <TKey, TPayload>(
this QueryContainer container,
Stream binaryStream,
IIngressScheduler scheduler = null,
bool readPropertiesFromStream = false,
StreamProperties <TKey, TPayload> inputProperties = null,
string identifier = null)
{
return(new BinaryIngressStreamable <TKey, TPayload>(
binaryStream,
0,
scheduler,
inputProperties,
readPropertiesFromStream,
container,
identifier ?? Guid.NewGuid().ToString()));
}
/// <summary>
/// Marks a position into the stream
/// </summary>
/// <param name="index">The position that will be marked</param>
/// <param name="stream">The stream to mark</param>
public void MarkPosition(int index, System.IO.Stream stream)
{
if (this[stream] == null)
{
StreamProperties tempProps = new StreamProperties();
tempProps.marklimit = index;
tempProps.markposition = stream.Position;
Add(stream, tempProps);
}
else
{
((StreamProperties)this[stream]).marklimit = index;
((StreamProperties)this[stream]).markposition = stream.Position;
}
}
