internal void Write(Emitter <Message <BufferReader> > source, PsiStreamMetadata meta, DeliveryPolicy deliveryPolicy = null)
{
var mergeInput = this.merger.Add(meta.Name); // this checks for duplicates
this.writer.OpenStream(meta);
Operators.PipeTo(source, mergeInput, true, deliveryPolicy);
}
private void OpenStream <T>(PsiStreamMetadata meta, Action <T, Envelope> target, Func <T> allocator = null)
{
OpenedStreamIds.Add(meta.Id);
// Get the deserialization handler for this stream type
var handler = this.serializers.GetHandler <T>();
// If there's no list of targets for this stream, create it now
if (!this.targets.ContainsKey(meta.Id))
{
this.targets[meta.Id] = new List <Delegate>();
}
// Add the target to the list to call when this stream has new data
this.targets[meta.Id].Add(target);
// Update the code to execute when this stream receives new data
this.outputs[meta.Id] = (br, e) =>
{
// Deserialize the data
var data = this.Deserialize <T>(handler, br, (allocator == null) ? default(T) : allocator());
// Call each of the targets
foreach (Delegate action in this.targets[meta.Id])
{
(action as Action <T, Envelope>)(data, e);
}
};
}
/// <summary>
/// Creates a logical storage stream to write messages to.
/// </summary>
/// <param name="streamId">The id of the stream, unique for this store. All messages with this stream id will be written to this storage stream.</param>
/// <param name="streamName">The name of the stream. This name can be later used to open the sorage stream for reading.</param>
/// <param name="indexed">Indicates whether the stream is indexed or not. Indexed streams have a small index entry in the main data file and the actual message body in a large data file.</param>
/// <param name="typeName">A name identifying the type of the messages in this stream. This is usually a fully-qualified type name or a data contract name, but can be anything that the caller wants.</param>
/// <returns>The complete metadata for the storage stream just created.</returns>
public PsiStreamMetadata OpenStream(int streamId, string streamName, bool indexed, string typeName)
{
if (this.metadata.ContainsKey(streamId))
{
throw new InvalidOperationException($"The stream id {streamId} has already been registered with this writer.");
}
var meta = new PsiStreamMetadata(streamName, streamId, typeName);
meta.Opened = Time.GetCurrentTime();
meta.IsPersisted = true;
meta.IsIndexed = indexed;
meta.PartitionName = this.name;
meta.PartitionPath = this.path;
this.metadata[streamId] = meta;
this.WriteToCatalog(meta);
// make sure we have a large file if needed
if (indexed)
{
this.largeMessageWriter = this.largeMessageWriter ?? new MessageWriter(StoreCommon.GetLargeDataFileName(this.name), this.path, this.append);
}
return(meta);
}
private void OpenStream <T>(PsiStreamMetadata meta, Action <T, Envelope> target, Func <T> allocator = null)
{
OpenedStreamIds.Add(meta.Id);
var handler = this.serializers.GetHandler <T>();
this.outputs[meta.Id] = (br, e) => target(this.Deserialize <T>(handler, br, (allocator == null) ? default(T) : allocator()), e);
}
public bool TryGet(string name, out PsiStreamMetadata metadata)
{
if (!this.streamDescriptors.ContainsKey(name) || !this.streamDescriptors[name].IsClosed)
{
this.Update();
}
return(this.streamDescriptors.TryGetValue(name, out metadata));
}
/// <summary>
/// Opens the specified stream as raw `Message` of `BufferReader` for reading and returns a stream instance that can be used to consume the messages.
/// The returned stream will publish data read from the store once the pipeline is running.
/// </summary>
/// <remarks>Messages are not deserialized.</remarks>
/// <param name="meta">The meta of the stream to open.</param>
/// <returns>A stream of raw messages that publishes the data read from the store.</returns>
internal IProducer <Message <BufferReader> > OpenRawStream(PsiStreamMetadata meta)
{
if (this.streamReader is not PsiStoreStreamReader)
{
throw new NotSupportedException($"Opening raw streams requires a {nameof(PsiStoreStreamReader)}.");
}
return(this.OpenStream <Message <BufferReader> >(meta.Name));
}
public bool TryGet(int id, out PsiStreamMetadata metadata)
{
if (!this.streamDescriptorsById.ContainsKey(id) || !this.streamDescriptorsById[id].IsClosed)
{
this.Update();
}
return(this.streamDescriptorsById.TryGetValue(id, out metadata));
}
/// <summary>
/// Opens the specified storage stream as raw `Message` of `BufferReader` for reading and returns a stream instance that can be used to consume the messages.
/// The returned stream will publish data read from the store once the pipeline is running.
/// </summary>
/// <remarks>Messages are not deserialized.</remarks>
/// <param name="meta">The meta of the storage stream to open.</param>
/// <returns>A stream of raw messages that publishes the data read from the store.</returns>
internal Emitter <Message <BufferReader> > OpenRawStream(PsiStreamMetadata meta)
{
if (meta.OriginatingLifetime != null && !meta.OriginatingLifetime.IsEmpty && meta.OriginatingLifetime.IsFinite)
{
// propose a replay time that covers the stream lifetime
this.ProposeReplayTime(meta.OriginatingLifetime);
}
this.reader.OpenStream(meta); // this checks for duplicates but bypasses type checks
return(this.splitter.Add(meta.Id));
}
/// <summary>
/// Opens the specified stream as raw `Message` of `BufferReader` for reading and returns a stream instance that can be used to consume the messages.
/// The returned stream will publish data read from the store once the pipeline is running.
/// </summary>
/// <remarks>Messages are not deserialized.</remarks>
/// <param name="meta">The meta of the stream to open.</param>
/// <returns>A stream of raw messages that publishes the data read from the store.</returns>
internal IProducer <Message <BufferReader> > OpenRawStream(PsiStreamMetadata meta)
{
var storeStreamReader = this.streamReader as PsiStoreStreamReader;
if (storeStreamReader == null)
{
throw new NotSupportedException("Opening raw streams only words with PsiStoreStreamReader as underlying IStreamReader.");
}
return(this.OpenStream <Message <BufferReader> >(meta.Name));
}
/// <summary>
/// Opens the specified stream for reading.
/// </summary>
/// <param name="meta">The metadata describing the stream to open.</param>
/// <returns>True if the stream was successfully opened, false if no matching stream could be found.</returns>
public bool OpenStream(PsiStreamMetadata meta)
{
if (this.enabledStreams.Contains(meta.Id))
{
return(false);
}
this.enabledStreams.Add(meta.Id);
this.IsIndexedStream(meta.Id); // update `isIndexedStream` dictionary
return(true);
}
internal void Write(Emitter <Message <BufferReader> > source, PsiStreamMetadata meta, DeliveryPolicy deliveryPolicy = null)
{
var mergeInput = this.merger.Add(meta.Name); // this checks for duplicates
var connector = this.CreateInputConnectorFrom <Message <BufferReader> >(source.Pipeline, null);
source.PipeTo(connector);
source.Name = source.Name ?? meta.Name;
connector.Out.Name = meta.Name;
this.writer.OpenStream(meta);
connector.PipeTo(mergeInput, true, deliveryPolicy);
}
internal void Write(Emitter <Message <BufferReader> > source, PsiStreamMetadata meta, DeliveryPolicy <Message <BufferReader> > deliveryPolicy = null)
{
var mergeInput = this.merger.Add(meta.Name); // this checks for duplicates
var connector = this.CreateInputConnectorFrom <Message <BufferReader> >(source.Pipeline, null);
source.PipeTo(connector);
source.Name ??= meta.Name;
connector.Out.Name = meta.Name;
this.writer.OpenStream(meta);
// defaults to lossless delivery policy unless otherwise specified
connector.PipeTo(mergeInput, true, deliveryPolicy ?? DeliveryPolicy.Unlimited);
}
/// <summary>
/// Attempt to get stream metadata (available once stream has been opened).
/// </summary>
/// <param name="streamId">The id of the stream, unique for this store.</param>
/// <param name="meta">The metadata for the stream, if it has previously been opened.</param>
/// <returns>True if stream metadata if stream has been opened so that metadata is available.</returns>
public bool TryGetMetadata(int streamId, out PsiStreamMetadata meta)
{
return(this.metadata.TryGetValue(streamId, out meta));
}
/// <summary>
/// Opens the specified stream as raw `Message` of `BufferReader` for reading and returns a stream instance that can be used to consume the messages.
/// The returned stream will publish data read from the store once the pipeline is running.
/// </summary>
/// <remarks>Messages are not deserialized.</remarks>
/// <param name="meta">The meta of the stream to open.</param>
/// <returns>A stream of raw messages that publishes the data read from the store.</returns>
internal IProducer <Message <BufferReader> > OpenRawStream(PsiStreamMetadata meta)
{
return(this.BridgeOut(this.getStreamImporter().OpenRawStream(meta), meta.Name));
}
/// <summary>
/// Creates a stream to write messages to.
/// The stream characteristics are extracted from the provided metadata descriptor.
/// </summary>
/// <param name="meta">The metadata describing the stream to open.</param>
/// <returns>The complete metadata for the stream just created.</returns>
public PsiStreamMetadata OpenStream(PsiStreamMetadata meta)
{
return(this.OpenStream(meta.Id, meta.Name, meta.IsIndexed, meta.TypeName).UpdateSupplementalMetadataFrom(meta));
}
/// <summary>
/// Creates a logical storage stream to write messages to.
/// The storage stream characteristics are extracted form the prvoided metadata descriptor.
/// </summary>
/// <param name="meta">The metadata describing the stream to open.</param>
/// <returns>The complete metadata for the storage stream just created.</returns>
public PsiStreamMetadata OpenStream(PsiStreamMetadata meta)
{
return(this.OpenStream(meta.Id, meta.Name, meta.IsIndexed, meta.TypeName));
}
/// <summary>
/// Writes the messages from the specified stream to the matching stream in this store.
/// </summary>
/// <typeparam name="TMessage">The type of messages in the stream.</typeparam>
/// <param name="source">The source stream to write.</param>
/// <param name="name">The name of the stream.</param>
/// <param name="metadata">Source stream metadata.</param>
/// <param name="deliveryPolicy">An optional delivery policy.</param>
internal void Write <TMessage>(Emitter <TMessage> source, string name, PsiStreamMetadata metadata, DeliveryPolicy <TMessage> deliveryPolicy = null)
{
this.WriteToStorage(source, name, metadata.IsIndexed, deliveryPolicy).UpdateSupplementalMetadataFrom(metadata);
}
/// <summary>
/// Opens the specified storage stream as raw `Message` of `BufferReader` for reading and returns a stream instance that can be used to consume the messages.
/// The returned stream will publish data read from the store once the pipeline is running.
/// </summary>
/// <remarks>Messages are not deserialized.</remarks>
/// <param name="meta">The meta of the storage stream to open.</param>
/// <returns>A stream of raw messages that publishes the data read from the store.</returns>
internal Emitter <Message <BufferReader> > OpenRawStream(PsiStreamMetadata meta)
{
this.reader.OpenStream(meta); // this checks for duplicates but bypasses type checks
return(this.splitter.Add(meta.Id));
}
