public SnapshotWindowStreamable(IStreamable <TKey, TInput> source, IAggregate <TInput, TState, TOutput> aggregate) : base(source, source.Properties.Snapshot(aggregate)) { Contract.Requires(source != null); this.Aggregate = aggregate; this.sourceProps = source.Properties; if (this.sourceProps.IsStartEdgeOnly) { this.apt = AggregatePipeType.StartEdge; } else if (this.sourceProps.IsConstantDuration) { if (this.sourceProps.IsTumbling) { this.apt = AggregatePipeType.Tumbling; } else if (this.sourceProps.ConstantDurationLength.HasValue && this.sourceProps.IsConstantHop && this.sourceProps.ConstantHopLength.HasValue) { this.apt = AggregatePipeType.Hopping; } else { this.apt = AggregatePipeType.Sliding; } } else { this.apt = AggregatePipeType.PriorityQueue; } Initialize(); }
/// <summary> /// Ungroup with group-selector /// </summary> internal static StreamProperties <TOuterKey, TResult> Ungroup <TOuterKey, TInnerKey, TPayload, TResult>( this StreamProperties <CompoundGroupKey <TOuterKey, TInnerKey>, TPayload> source, Expression <Func <TInnerKey, TPayload, TResult> > groupSelector) { IEqualityComparerExpression <TOuterKey> newKeyEqualityComparer = null; if (source.KeyEqualityComparer != null) { newKeyEqualityComparer = ((CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>)source.KeyEqualityComparer).outerComparer; } IComparerExpression <TOuterKey> newKeyComparer = null; if (source.KeyComparer != null) { newKeyComparer = ((CompoundGroupKeyComparer <TOuterKey, TInnerKey>)source.KeyComparer).outerComparer; } return(new StreamProperties <TOuterKey, TResult>( source.IsColumnar, source.IsConstantDuration, source.ConstantDurationLength, source.IsConstantHop, source.ConstantHopLength, source.ConstantHopOffset, source.IsIntervalFree, false, false, false, newKeyEqualityComparer, EqualityComparerExpression <TResult> .Default, newKeyComparer, null, new Dictionary <Expression, object>(), new Dictionary <Expression, Guid?>(), source.QueryContainer)); }
void UpdatingForExistingStream() { var partition = new Partition(Table, Id + ".c"); var properties = new Dictionary <string, EntityProperty> { { "Created", new EntityProperty(DateTimeOffset.Now) }, { "Active", new EntityProperty(true) } }; Stream.Provision(partition, StreamProperties.From(properties)); Console.WriteLine("Stream metadata specified for stream in partition '{0}'", partition); var stream = Stream.Open(partition); Print(stream.Properties); properties["Active"] = new EntityProperty(false); Stream.SetProperties(stream, StreamProperties.From(properties)); Console.WriteLine("Updated stream metadata in partition '{0}'", partition); stream = Stream.Open(partition); Print(stream.Properties); }
public async void When_set_successfully() { var properties = new Dictionary <string, EntityProperty> { { "P1", new EntityProperty(42) }, { "P2", new EntityProperty("42") } }; var stream = await Stream.ProvisionAsync(partition, StreamProperties.From(properties)); var newProperties = new Dictionary <string, EntityProperty> { { "P1", new EntityProperty(56) }, { "P2", new EntityProperty("56") } }; var newStream = await Stream.SetPropertiesAsync(stream, StreamProperties.From(newProperties)); StreamProperties.From(newProperties).ToExpectedObject().ShouldEqual(newStream.Properties); var storedEntity = partition.RetrieveStreamEntity(); var storedProperties = storedEntity.Properties; StreamProperties.From(newProperties).ToExpectedObject().ShouldEqual(storedProperties); }
public async Task When_writing_to_existing_stream_non_null_stream_properties() { var properties = new { Property1 = "Foo", Property2 = 42 }; var stream = new Stream(partition, StreamProperties.From(properties)); var result = await Stream.WriteAsync(stream, CreateEvent("e1"), CreateEvent("e2")); stream = result.Stream; AssertNewStream(result, 2, properties); AssertStreamEntity(2, properties); var replaced = new { Property1 = "Bar" }; var restored = Stream.From(partition, stream.ETag, stream.Version, StreamProperties.From(replaced)); result = await Stream.WriteAsync(restored, CreateEvent("e3")); AssertNewStream(result, 3, replaced); AssertStreamEntity(3, replaced); }
private void TestStreamPropertiesClone(StreamProperties sp) { // Set members in source object sp.ContentType = "foo_1"; sp.ContentEncoding = "foo_2"; sp.Headers = new Dictionary <string, object>(); sp.Headers.Add("foo_3", "foo_4"); sp.Headers.Add("foo_5", "foo_6"); sp.Priority = 12; sp.Timestamp = new AmqpTimestamp(123); // Clone StreamProperties spClone = sp.Clone() as StreamProperties; // Change values in source object sp.ContentType = "foo_7"; sp.ContentEncoding = "foo_8"; sp.Headers.Remove("foo_3"); sp.Headers.Remove("foo_5"); sp.Headers.Add("foo_9", "foo_10"); sp.Headers.Add("foo_11", "foo_12"); sp.Priority = 34; sp.Timestamp = new AmqpTimestamp(234); // Make sure values have not changed in clone Assert.AreEqual("foo_1", spClone.ContentType); Assert.AreEqual("foo_2", spClone.ContentEncoding); Assert.AreEqual(2, spClone.Headers.Count); Assert.AreEqual(true, spClone.Headers.ContainsKey("foo_3")); Assert.AreEqual("foo_4", spClone.Headers["foo_3"]); Assert.AreEqual(true, spClone.Headers.ContainsKey("foo_5")); Assert.AreEqual("foo_6", spClone.Headers["foo_5"]); Assert.AreEqual(12, spClone.Priority); Assert.AreEqual(new AmqpTimestamp(123), spClone.Timestamp); }
private StreamProperties GetPropertiesFromStream(Stream stream, bool isImage) { StreamProperties streamProperties = new StreamProperties { FileSize = 0, ImageHeight = 0, ImageWidth = 0 }; using (MemoryStream memoryStream = new MemoryStream()) { stream.CopyTo(memoryStream); memoryStream.Position = 0; streamProperties.FileSize = memoryStream.Length; if (isImage) { try { using (Image image = Image.FromStream(memoryStream)) { streamProperties.ImageWidth = image.Width; streamProperties.ImageHeight = image.Height; } } catch (Exception ex) { EventLog.Application.WriteWarning("Failed to get image from stream. Image entity width and height will not be set. Exception {0}.", ex.ToString()); streamProperties.ImageWidth = 0; streamProperties.ImageHeight = 0; } } return(streamProperties); } }
public ShardedCacheObserver(StreamCache <TKey, TPayload> cache, StreamProperties <TKey, TPayload> sourceProps) { this.cache = cache; this.sourceProps = sourceProps; this.elements = new List <QueuedMessage <StreamMessage <TKey, TPayload> > >(); }
public async Task When_provisioning_along_with_custom_properties() { var properties = new Dictionary <string, EntityProperty> { { "Created", new EntityProperty(DateTimeOffset.Now) }, { "Active", new EntityProperty(true) } }; var stream = await Stream.ProvisionAsync(partition, StreamProperties.From(properties)); var entity = partition.RetrieveStreamEntity(); var expectedStream = new Stream ( partition, entity.ETag, 0, StreamProperties.From(properties) ); expectedStream.ToExpectedObject().ShouldEqual(stream); var expectedEntity = new { RowKey = Api.StreamRowKey, Properties = StreamProperties.From(properties), Version = 0 }; expectedEntity.ToExpectedObject().ShouldMatch(entity); }
Stream CreateStream(int version, string etag, object properties = null) { var props = properties != null ? StreamProperties.From(properties) : StreamProperties.None; return(new Stream(partition, etag, version, props)); }
protected ConnectableStreamableBase(StreamProperties <TKey, TPayload> properties) : base(properties) { Contract.Requires(properties != null); this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(properties.IsColumnar); this.ClassId = Guid.NewGuid(); }
public BinaryStreamObserver(StreamProperties <TKey, TPayload> streamProperties, Stream stream) { this.serializer = StreamSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >( new SerializerSettings() { KnownTypes = StreamMessageManager.GeneratedTypes() }); this.stream = stream; }
public void When_passing_property_with_reserved_name() { var reserved = ReservedStreamProperties() .ToDictionary(p => p, p => new EntityProperty(42)); var properties = StreamProperties.From(reserved); Assert.That(properties.Count, Is.EqualTo(0), "Should skip all properties with reserved names, such as RowKey, Id, etc"); }
Map2Reduce <TMapInputLeft1, TMapInputRight1, TMapInputLeft2, TMapInputRight2, TReduceKey, TReduceInput1, TReduceInput2, TBind, TOutput>( this StreamProperties <Empty, TMapInputLeft1> leftStream1, StreamProperties <Empty, TMapInputRight1> rightStream1, StreamProperties <Empty, TMapInputLeft2> leftStream2, StreamProperties <Empty, TMapInputRight2> rightStream2, Func <IStreamable <Empty, TMapInputLeft1>, IStreamable <Empty, TMapInputRight1>, IStreamable <Empty, TReduceInput1> > mapper1, Func <IStreamable <Empty, TMapInputLeft2>, IStreamable <Empty, TMapInputRight2>, IStreamable <Empty, TReduceInput2> > mapper2, Func <IStreamable <TReduceKey, TReduceInput1>, IStreamable <TReduceKey, TReduceInput2>, IStreamable <TReduceKey, TBind> > reducer, Expression <Func <TReduceInput1, TReduceKey> > keySelector1, Expression <Func <TReduceInput2, TReduceKey> > keySelector2, Expression <Func <TReduceKey, TBind, TOutput> > resultSelector, OperationalHint reduceOptions) { var map1 = leftStream1.Derive(a => mapper1(a, null)); var map2 = leftStream2.Derive(a => mapper2(a, null)); var group1 = map1.Group(keySelector1); var group2 = map2.Group(keySelector2); if (leftStream1.CanSpray(leftStream2, keySelector1, keySelector2) && group1.CanSpray(group2, keySelector1, keySelector2)) { var reduce = group1.Derive(group2, reducer).Ungroup(resultSelector); if (Config.MapArity > 1) { reduce = reduce.Union(reduce); } return(reduce); } else { if (Config.MapArity > 1) { group1 = group1.Union(group1); group2 = group2.Union(group2); } var reduce = group1.Derive(group2, reducer).Ungroup(resultSelector); if (Config.ReduceArity > 1) { reduce = reduce.Union(reduce); } return(reduce); } }
protected BinaryStreamable(StreamProperties <TKey, TResult> properties, IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right, bool registerInputs = false) : base(properties) { Contract.Requires(left != null); Contract.Requires(right != null); this.Left = left; this.Right = right; this.registerInputs = registerInputs; }
public async void When_property_map_is_empty() { var properties = new Dictionary <string, EntityProperty>(); var previous = await Stream.ProvisionAsync(partition); var current = await Stream.SetPropertiesAsync(previous, StreamProperties.From(properties)); Assert.That(current.ETag, Is.Not.EqualTo(previous.ETag)); StreamProperties.From(properties).ToExpectedObject().ShouldEqual(current.Properties); }
/// <summary> /// Map + single-input Reduce /// </summary> internal static StreamProperties <TKey, TOutput> MapReduce <TKey, TMapInputLeft, TMapInputRight, TReduceInput, TReduceKey, TBind, TOutput>( this StreamProperties <TKey, TMapInputLeft> leftStream, StreamProperties <TKey, TMapInputRight> rightStream, Func <IStreamable <TKey, TMapInputLeft>, IStreamable <TKey, TMapInputRight>, IStreamable <TKey, TReduceInput> > mapper, Expression <Func <TReduceInput, TReduceKey> > keySelector, Func <IStreamable <CompoundGroupKey <TKey, TReduceKey>, TReduceInput>, IStreamable <CompoundGroupKey <TKey, TReduceKey>, TBind> > reducer, Expression <Func <TReduceKey, TBind, TOutput> > resultSelector) { bool mc = leftStream.IsMulticore; var map = leftStream.ToMulticore(true).Derive(a => mapper(a, null)); if (reducer != null) { // We need to test the reducer function to see what it does to properties. var group = map.GroupNested(keySelector); if (group.CanSpray(keySelector)) { var reduce = group.Derive(reducer).Ungroup(resultSelector); if (Config.MapArity > 1) { reduce = reduce.Union(reduce); } var returnValue = reduce.ToMulticore(mc); return(returnValue); } else { if (Config.MapArity > 1) { group = group.Union(group); } var reduce = group.Derive(reducer).Ungroup(resultSelector); if (Config.ReduceArity > 1) { reduce = reduce.Union(reduce); } var returnValue = reduce.ToMulticore(mc); return(returnValue); } } else { var result = leftStream.Derive(a => mapper(a, null)) as StreamProperties <TKey, TOutput>; if (Config.MapArity > 1) { result = result.Union(result); } return(result.ToMulticore(mc)); } }
// TODO: This appears to be copied code from Binary egress - can we unify? public ShardedSerializerObserver(Stream destination, StreamProperties <TKey, TPayload> sourceProps, bool writePropertiesToStream = false) { this.destination = destination; if (writePropertiesToStream) { var propSer = StreamableSerializer.Create <SerializedProperties>(); propSer.Serialize(destination, SerializedProperties.FromStreamProperties(sourceProps)); } this.serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(new SerializerSettings()); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TPayload"></typeparam> /// <param name="fileName"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <returns></returns> public static IPassiveIngressStreamable <TKey, TPayload> ToStreamablePassiveFromFile <TKey, TPayload>( this string fileName, bool readPropertiesFromStream = false, StreamProperties <TKey, TPayload> inputProperties = null) { return(new BinaryIngressStreamablePassive <TKey, TPayload>( new FileStream(fileName, FileMode.Open), inputProperties, readPropertiesFromStream, null, Guid.NewGuid().ToString())); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TPayload"></typeparam> /// <param name="binaryStream"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <returns></returns> public static IPassiveIngressStreamable <TKey, TPayload> ToStreamablePassive <TKey, TPayload>( this Stream binaryStream, bool readPropertiesFromStream = false, StreamProperties <TKey, TPayload> inputProperties = null) { return(new BinaryIngressStreamablePassive <TKey, TPayload>( binaryStream, inputProperties, readPropertiesFromStream, null, Guid.NewGuid().ToString())); }
public static SerializedProperties FromStreamProperties <TKey, TPayload>(StreamProperties <TKey, TPayload> props) => new SerializedProperties { IsColumnar = props.IsColumnar, IsConstantDuration = props.IsConstantDuration, ConstantDurationLength = props.ConstantDurationLength, IsConstantHop = props.IsConstantHop, ConstantHopLength = props.ConstantHopLength, ConstantHopOffset = props.ConstantHopOffset, IsIntervalFree = props.IsIntervalFree, IsSyncTimeSimultaneityFree = props.IsSyncTimeSimultaneityFree, IsEventOverlappingFree = props.IsEventOverlappingFree };
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TPayload"></typeparam> /// <param name="fileName"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <param name="container"></param> /// <param name="identifier"></param> /// <returns></returns> public static IPassiveIngressStreamable <TKey, TPayload> RegisterBinaryInputPassiveFromFile <TKey, TPayload>( this QueryContainer container, string fileName, bool readPropertiesFromStream = false, StreamProperties <TKey, TPayload> inputProperties = null, string identifier = null) { return(new BinaryIngressStreamablePassive <TKey, TPayload>( new FileStream(fileName, FileMode.Open), inputProperties, readPropertiesFromStream, container, identifier ?? Guid.NewGuid().ToString())); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TPayload"></typeparam> /// <param name="binaryStream"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <param name="container"></param> /// <param name="identifier"></param> /// <returns></returns> public static IPassiveIngressStreamable <Empty, TPayload> RegisterBinaryInputPassive <TPayload>( this QueryContainer container, Stream binaryStream, bool readPropertiesFromStream = false, StreamProperties <Empty, TPayload> inputProperties = null, string identifier = null) { return(new BinaryIngressStreamablePassive <Empty, TPayload>( binaryStream, inputProperties, readPropertiesFromStream, container, identifier ?? Guid.NewGuid().ToString())); }
public StreamManagerTest() { _mockEventSource = new Mock <IEventSource>(); _mockEventSource.Setup(es => es.StartAsync()).Returns(Task.CompletedTask); _eventSource = _mockEventSource.Object; _eventSourceCreator = new StubEventSourceCreator(_eventSource); _config = new SimpleConfiguration { SdkKey = SdkKey }; _mockStreamProcessor = new Mock <IStreamProcessor>(); _streamProcessor = _mockStreamProcessor.Object; _streamProperties = new StreamProperties(StreamUri, HttpMethod.Get, null); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TPayload"></typeparam> /// <param name="binaryStream"></param> /// <param name="scheduler"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <returns></returns> public static IIngressStreamable <Empty, TPayload> ToStreamable <TPayload>( this Stream binaryStream, IIngressScheduler scheduler = null, bool readPropertiesFromStream = false, StreamProperties <Empty, TPayload> inputProperties = null) { return(new BinaryIngressStreamable <Empty, TPayload>( binaryStream, 0, scheduler, inputProperties, readPropertiesFromStream, null, Guid.NewGuid().ToString())); }
void AssertStreamEntity(int version = 0, object properties = null) { var newStreamEntity = partition.RetrieveStreamEntity(); var expectedEntity = new { RowKey = Api.StreamRowKey, Properties = properties != null ? StreamProperties.From(properties) : StreamProperties.None, Version = version, }; newStreamEntity.ShouldMatch(expectedEntity.ToExpectedObject()); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TPayload"></typeparam> /// <param name="fileName"></param> /// <param name="numMessages"></param> /// <param name="scheduler"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <returns></returns> public static IIngressStreamable <Empty, TPayload> ToStreamableFromFile <TPayload>( this string fileName, int numMessages = 0, IIngressScheduler scheduler = null, bool readPropertiesFromStream = false, StreamProperties <Empty, TPayload> inputProperties = null) { return(new BinaryIngressStreamable <Empty, TPayload>( new FileStream(fileName, FileMode.Open), numMessages, scheduler, inputProperties, readPropertiesFromStream, null, Guid.NewGuid().ToString())); }
void SimultaneousSettingOfStreamMetadata() { var a = Stream.Open(Partition); var b = Stream.Open(Partition); Stream.SetProperties(a, StreamProperties.From(new { A = 42 })); try { Stream.SetProperties(b, StreamProperties.From(new { A = 56 })); } catch (ConcurrencyConflictException) { Console.WriteLine("Simultaneously setting metadata using the same version of stream will lead to ConcurrencyConflictException"); } }
public GroupedWindowStreamable(IStreamable <Empty, TInput> source, IAggregate <TInput, TState, TOutput> aggregate, Expression <Func <TInput, TKey> > keySelector, Expression <Func <TKey, TOutput, TResult> > resultSelector) : base(source, source.Properties.Group(keySelector).Snapshot(aggregate).Ungroup(resultSelector)) { Contract.Requires(source != null); this.Aggregate = aggregate; this.sourceProps = source.Properties; this.KeySelector = keySelector; this.ResultSelector = resultSelector; if (!this.sourceProps.IsStartEdgeOnly) { throw new InvalidOperationException("Cannot use this streamable if the input stream is not guaranteed to be start-edge only."); } Initialize(); }
/// <summary> /// Deserialize from binary stream to streamable /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TPayload"></typeparam> /// <param name="binaryStream"></param> /// <param name="scheduler"></param> /// <param name="readPropertiesFromStream"></param> /// <param name="inputProperties"></param> /// <param name="container"></param> /// <param name="identifier"></param> /// <returns></returns> public static IIngressStreamable <TKey, TPayload> RegisterBinaryInput <TKey, TPayload>( this QueryContainer container, Stream binaryStream, IIngressScheduler scheduler = null, bool readPropertiesFromStream = false, StreamProperties <TKey, TPayload> inputProperties = null, string identifier = null) { return(new BinaryIngressStreamable <TKey, TPayload>( binaryStream, 0, scheduler, inputProperties, readPropertiesFromStream, container, identifier ?? Guid.NewGuid().ToString())); }
/// <summary> /// Marks a position into the stream /// </summary> /// <param name="index">The position that will be marked</param> /// <param name="stream">The stream to mark</param> public void MarkPosition(int index, System.IO.Stream stream) { if (this[stream] == null) { StreamProperties tempProps = new StreamProperties(); tempProps.marklimit = index; tempProps.markposition = stream.Position; Add(stream, tempProps); } else { ((StreamProperties)this[stream]).marklimit = index; ((StreamProperties)this[stream]).markposition = stream.Position; } }