/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
threads = new IThread[configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < configuration.NumStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
public override List <StreamState> GetStreamsInFile()
{
if (CurrentDoc != null)
{
DocumentStreams = StreamStateManager.ReadState(CurrentDoc.Document);
}
return(DocumentStreams);
}
private void CheckForStreamsAndOpenSpeckle(object sender, Autodesk.Revit.DB.Events.DocumentOpenedEventArgs e)
{
var streams = StreamStateManager.ReadState(e.Document);
if (streams != null && streams.Count != 0)
{
SpeckleRevitCommand.OpenOrFocusSpeckle(AppInstance);
}
}
//public override void RemoveStreamFromFile(string streamId)
//{
// var streamState = DocumentStreams.FirstOrDefault(s => s.Stream.id == streamId);
// if (streamState != null)
// {
// DocumentStreams.Remove(streamState);
// WriteStateToFile();
// }
//}
public override List <StreamState> GetStreamsInFile()
{
if (Model != null)
{
DocumentStreams = StreamStateManager.ReadState(Model);
}
return(DocumentStreams);
}
/// <summary>
/// Transaction wrapper around writing the local streams to the file.
/// </summary>
private void WriteStateToFile()
{
if (Control.InvokeRequired)
{
Control.Invoke(new WriteStateDelegate(StreamStateManager.WriteStreamStateList), new object[] { DocumentStreams });
}
else
{
StreamStateManager.WriteStreamStateList(DocumentStreams);
}
}
public override void WriteStreamsToFile(List <StreamState> streams)
{
StreamStateManager.ClearStreamStateList(Model);
foreach (var s in streams)
{
DocumentStreams.Add(s);
WriteStateToFile();
}
//throw new NotImplementedException();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = kafkaSupplier;
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(this.threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
this.queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
/// <summary>
/// Transaction wrapper around writing the local streams to the file.
/// </summary>
private void WriteStateToFile()
{
Queue.Add(new Action(() =>
{
using (Transaction t = new Transaction(CurrentDoc.Document, "Speckle Write State"))
{
t.Start();
StreamStateManager.WriteStreamStateList(CurrentDoc.Document, DocumentStreams);
t.Commit();
}
}));
Executor.Raise();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance.
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
public KafkaStream(Topology topology, IStreamConfig configuration)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = new DefaultKafkaClientSupplier(new KafkaLoggerAdapter(configuration));
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
StreamState = State.CREATED;
}
public override List <StreamState> GetStreamsInFile()
{
if (File != null)
{
ECSchema schema;
if (Control.InvokeRequired)
{
schema = (ECSchema)Control.Invoke(new ListSchemaDelegate(StreamStateManager.StreamStateListSchema.GetSchema));
DocumentStreams = (List <StreamState>)Control.Invoke(new ReadSchemaDelegate(StreamStateManager.ReadState), new object[] { schema });
}
else
{
schema = StreamStateManager.StreamStateListSchema.GetSchema();
if (schema == null)
{
schema = StreamStateManager.StreamStateListSchema.AddSchema();
}
DocumentStreams = StreamStateManager.ReadState(schema);
}
}
return(DocumentStreams);
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads;
threads = new IThread[numStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology();
bool hasGlobalTopology = globalTaskTopology != null;
if (numStreamThreads == 0 && !hasGlobalTopology)
{
throw new TopologyException("Topology has no stream threads and no global threads, " +
"must subscribe to at least one source topic or global table.");
}
GlobalThreadState globalThreadState = null;
if (hasGlobalTopology)
{
string globalThreadId = $"{clientId}-GlobalStreamThread";
GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology,
globalThreadId,
kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)),
configuration,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId)));
globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread();
globalThreadState = globalStreamThread.State;
}
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < numStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState, globalThreadState);
if (hasGlobalTopology)
{
globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange;
}
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores);
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider);
StreamState = State.CREATED;
}
//public override void AddNewStream(StreamState state)
//{
// Tracker.TrackPageview(Tracker.STREAM_CREATE);
// var index = DocumentStreams.FindIndex(b => b.Stream.id == state.Stream.id);
// if (index == -1)
// {
// DocumentStreams.Add(state);
// WriteStateToFile();
// }
//}
private void WriteStateToFile()
{
StreamStateManager.WriteStreamStateList(Model, DocumentStreams);
}
public override List<StreamState> GetStreamsInFile()
{
DocumentStreams = StreamStateManager.ReadState(CurrentDoc.Document);
return DocumentStreams;
}
