public SubscriptionMessageObserver(
IServiceProvider serviceProvider,
IStreamFilter filter)
{
_serviceProvider = serviceProvider;
_filter = filter;
}
public SimpleMessageStreamProvider(
string name,
SimpleMessageStreamProviderOptions options,
IStreamFilter streamFilter,
ILoggerFactory loggerFactory,
IServiceProvider services)
{
this.loggerFactory = loggerFactory;
this.Name = name;
this.logger = loggerFactory.CreateLogger <SimpleMessageStreamProvider>();
this.options = options;
this.streamFilter = streamFilter;
this.providerRuntime = services.GetRequiredService <IStreamProviderRuntime>();
this.runtimeClient = providerRuntime.ServiceProvider.GetService <IRuntimeClient>();
if (this.options.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.options.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>()
.GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
logger.Info(
"Initialized SimpleMessageStreamProvider with name {0} and with property FireAndForgetDelivery: {1}, OptimizeForImmutableData: {2} " +
"and PubSubType: {3}", Name, this.options.FireAndForgetDelivery, this.options.OptimizeForImmutableData,
this.options.PubSubType);
}
/// <summary>
/// Overrides the default implementation to encrypt the data, then write it to the underlying stream
/// </summary>
/// <param name="pfo"></param>
protected override void WriteIndirectStreamData(IIndirectObject pfo)
{
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.Begin(TraceLevel.Debug, "Secure Writer", "Encrypting stream for object '" + pfo.Number + " " + pfo.Generation + "'");
}
using (var mon = this._monitor.Record(PerformanceMonitorType.Encrypting_Streams, pfo.ToString()))
{
byte[] unencrypted = pfo.Stream.GetStreamData();
IStreamFilter enc = CreateEncryptionFilter(pfo.Number, pfo.Generation);
byte[] encrypted = enc.FilterStream(unencrypted);
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.Add(TraceLevel.Debug, "Encryption", "Encrypted stream from original " + unencrypted.Length + "bytes, now writing " + encrypted.Length + " encypted data bytes");
}
this.BaseStream.Write(Constants.StartStream);
this.BaseStream.Write(encrypted);
this.BaseStream.Write(Constants.EndStream);
}
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.End(TraceLevel.Debug, "Secure Writer", "Encrypting stream for object '" + pfo.Number + " " + pfo.Generation + "'");
}
else if (this.TraceLog.ShouldLog(TraceLevel.Verbose))
{
this.TraceLog.Add(TraceLevel.Verbose, "Encryption", "Encrypted the stream data for " + pfo.ToString());
}
}
public TModel Execute(TModel newValue, TModel lastValue)
{
if (null == newValue)
{
throw new ArgumentNullException(nameof(newValue));
}
if (null == lastValue)
{
throw new ArgumentNullException(nameof(lastValue));
}
TModel value = newValue;
var filters = (IList <IStreamFilter <TModel> >)Filters;
for (int i = 0; i < filters.Count; i++)
{
IStreamFilter <TModel> filter = filters[i];
value = filter.Execute(value, lastValue);
if (value == null)
{
break;
}
}
return(value);
}
internal SimpleMessageStreamProducer(
StreamImpl <T> stream,
string streamProviderName,
IStreamProviderRuntime providerUtilities,
bool fireAndForgetDelivery,
bool optimizeForImmutableData,
IStreamPubSub pubSub,
IStreamFilter streamFilter,
bool isRewindable,
SerializationManager serializationManager,
ILogger <SimpleMessageStreamProducer <T> > logger)
{
this.stream = stream;
this.streamProviderName = streamProviderName;
providerRuntime = providerUtilities;
this.pubSub = pubSub;
this.streamFilter = streamFilter;
this.serializationManager = serializationManager;
connectedToRendezvous = false;
this.fireAndForgetDelivery = fireAndForgetDelivery;
this.optimizeForImmutableData = optimizeForImmutableData;
IsRewindable = isRewindable;
isDisposed = false;
initLock = new AsyncLock();
this.logger = logger;
ConnectToRendezvous().Ignore();
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.Info(ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
private void CompressFontFileData(byte[] orig)
{
IStreamFilter filter = PDFStreamFilters.FlateDecode;
this._filtereddata = filter.FilterStream(orig);
this._filterName = filter.FilterName;
}
internal Task DeliverItem(InternalStreamId streamId, IStreamFilter streamFilter, object item, bool fireAndForgetDelivery, bool optimizeForImmutableData)
{
var tasks = fireAndForgetDelivery ? null : new List <Task>();
foreach (var subscriptionKvp in consumers)
{
var remoteConsumer = subscriptionKvp.Value.StreamConsumer;
var filterData = subscriptionKvp.Value.FilterData;
try
{
if (!streamFilter.ShouldDeliver(streamId, item, filterData))
{
continue;
}
}
catch (Exception)
{
var message = $"Ignoring exception while trying to evaluate subscription filter '{streamFilter.GetType().Name}' with data '{filterData}' on stream {streamId}";
this.logger.LogWarning(message);
}
Task task = DeliverToRemote(remoteConsumer, streamId, subscriptionKvp.Key, item, optimizeForImmutableData, fireAndForgetDelivery);
if (fireAndForgetDelivery)
{
task.Ignore();
}
else
{
tasks.Add(task);
}
}
// If there's no subscriber, presumably we just drop the item on the floor
return(fireAndForgetDelivery ? Task.CompletedTask : Task.WhenAll(tasks));
}
internal StreamFactoryBase(IStreamFilter <TModel> filter)
{
if (null == filter)
{
throw new ArgumentNullException(nameof(filter));
}
_filter = filter;
_streams = new Dictionary <TId, Stream>();
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, IStreamPubSub pubsub, IStreamFilter streamFilter, ILogger logger, bool fireAndForget, bool optimizeForImmutable)
{
providerRuntime = providerRt;
streamPubSub = pubsub;
this.streamFilter = streamFilter;
fireAndForgetDelivery = fireAndForget;
optimizeForImmutableData = optimizeForImmutable;
remoteConsumers = new Dictionary <InternalStreamId, StreamConsumerExtensionCollection>();
this.logger = logger;
}
public StreamStore(IStreamFilter <TModel> filter)
{
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
_filter = filter;
_streams = new Dictionary <TIdentifier, Stream>();
}
public void TestInitialize()
{
fixture = new Fixture().Customize(new AutoMoqCustomization());
filter = Mock.Of <IStreamFilter <FakeModel> >();
Mock.Get(filter)
.Setup(x => x.Execute(It.IsAny <FakeModel>(), It.IsAny <FakeModel>()))
.Returns <FakeModel, FakeModel>((newValue, lastValue) => newValue);
sut = new StreamStore <string, FakeModel>(filter);
}
public void constructor_fails_if_filters_contains_null()
{
var filters = new IStreamFilter <FakeModel>[]
{
Mock.Of <IStreamFilter <FakeModel> >(),
null,
Mock.Of <IStreamFilter <FakeModel> >()
};
Action action = () => new CompositeStreamFilter <FakeModel>(filters);
action.ShouldThrow <ArgumentException>()
.Where(x => x.ParamName == "filters");
}
internal PersistentStreamPullingManager(
SystemTargetGrainId managerId,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
IStreamFilter streamFilter,
StreamPullingAgentOptions options,
ILoggerFactory loggerFactory,
SiloAddress siloAddress)
: base(managerId, siloAddress, lowPriority: false, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.options = options;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.streamFilter = streamFilter;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress,
IQueueAdapter queueAdapter,
IQueueAdapterCache queueAdapterCache,
IStreamFailureHandler streamFailureHandler)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));;
this.queueAdapterCache = queueAdapterCache;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
streamProviderName,
Silo,
QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
}
private byte[] ApplyFilters(IStreamFilter[] filters, byte[] input)
{
byte[] output = null;
try
{
for (int i = 0; i < filters.Length; i++)
{
IStreamFilter filter = this.Filters[i];
output = filter.FilterStream(input);
input = output;
}
}
catch (Exception ex)
{
string msg = string.Format(CommonErrors.CouldNotApplyStreamFilters, this.IndirectObject.ToString(), ex.Message);
throw new PDFStreamException(msg, ex);
}
return(output);
}
private TModel Filter(TModel model)
{
if (null == model)
{
return(null);
}
ThrowIfModelHasInvalidId(model);
IStreamFilter <TModel> filter = _factory.Filter;
TModel filtered = filter.Execute(model, _observable.Value);
if (null == filtered)
{
return(null);
}
ThrowIfModelHasInvalidId(filtered);
return(filtered);
}
/// <summary>
/// Overrides the default implementation to encrypt the data, then write it to the underlying stream
/// </summary>
/// <param name="pfo"></param>
protected override void WriteIndirectStreamData(IIndirectObject pfo)
{
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.Begin(TraceLevel.Debug, "Secure Writer", "Encrypting stream for object '" + pfo.Number + " " + pfo.Generation + "'");
}
byte[] unencrypted = pfo.Stream.GetStreamData();
IStreamFilter enc = CreateEncryptionFilter(pfo.Number, pfo.Generation);
byte[] encrypted = enc.FilterStream(unencrypted);
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.Add(TraceLevel.Debug, "Encryption", "Encrypted stream data, now writing");
}
this.BaseStream.Write(Constants.StartStream);
this.BaseStream.Write(encrypted);
this.BaseStream.Write(Constants.EndStream);
if (this.TraceLog.ShouldLog(TraceLevel.Debug))
{
this.TraceLog.End(TraceLevel.Debug, "Secure Writer", "Encrypting stream for object '" + pfo.Number + " " + pfo.Generation + "'");
}
}
/// <summary>
/// Adds the specified filter to be applied during playback.
/// </summary>
/// <param name="filter">The filter.</param>
public void Add(IStreamFilter filter)
{
_Filters.Add(filter);
}
public PartitionedProjectionObserver(IServiceProvider serviceProvider, IStreamFilter filter)
{
_filter = filter;
_serviceProvider = serviceProvider;
}
/// <summary>
/// Adds a new filter on the top of the chain
/// </summary>
/// <param name="filter">Stream filter to put on top of the chain</param>
/// <remarks>
/// This should *only* be called during initialization. We don't
/// support changing the filter change after the first read/write
/// has been done and it's not thread-safe to boot!
/// </remarks>
public void AddFilter(IStreamFilter filter)
{
_topStream = filter.CreateFilterStream(_topStream);
}
/// <summary>
/// Adds a new filter on the top of the chain
/// </summary>
/// <param name="filter">Stream filter to put on top of the chain</param>
/// <remarks>
/// This should *only* be called during initialization. We don't
/// support changing the filter change after the first read/write
/// has been done and it's not thread-safe to boot!
/// </remarks>
public void AddFilter(IStreamFilter filter)
{
_topStream = filter.CreateFilterStream(_topStream);
}
public StreamFactory(IStreamFilter <TModel> filter)
: base(filter)
{
}
public ConcurrentStreamFactory(IStreamFilter <TModel> filter)
: base(filter)
{
}
public ProjectionObserver(IServiceProvider serviceProvider, IStreamFilter streamFilter)
{
_serviceProvider = serviceProvider;
_filter = streamFilter;
}
public MyReader(IStreamFilter streamFilter)
{
this._streamFilter = streamFilter;
}
/// <summary>
/// Removes the specified filter.
/// </summary>
/// <param name="filter">The filter.</param>
public void Remove(IStreamFilter filter)
{
_Filters.Remove(filter);
}
