private bool SubmitMessage(EventProcessorInternal.IEventMessage message)
{
try
{
if (_messageQueue.TryAdd(message))
{
_inputCapacityExceeded.GetAndSet(false);
}
else
{
// This doesn't mean that the output event buffer is full, but rather that the main thread is
// seriously backed up with not-yet-processed events. We shouldn't see this.
if (!_inputCapacityExceeded.GetAndSet(true))
{
_logger.Warn("Events are being produced faster than they can be processed");
}
}
}
catch (InvalidOperationException)
{
// queue has been shut down
return(false);
}
return(true);
}
public IDisposable CreateSubscription(long?lastProcessedCheckpoint, Action <string, Exception> errorHandler)
{
_errorHandler = errorHandler;
var source = CreateSource(Offset.Sequence(lastProcessedCheckpoint ?? 0))
.Select(ee => (ee.Event as IDomainEvent, ee.Offset))
.Where(de => de.Item1 != null)
.Batch(20, de => ImmutableList <(IDomainEvent, Offset)> .Empty.Add(de !), (list, evt) => list.Add(evt !))
.Select(de =>
{
var(domainEvent, offset) = de.Last();
return(new Transaction
{
Checkpoint = ((Sequence)offset).Value,
Id = EventId.New.Value,
StreamId = domainEvent.GetIdentity().Value,
TimeStampUtc = domainEvent.Timestamp.DateTime,
Events = new List <LiquidProjections.EventEnvelope>(
de
.Select(pair =>
{
var(evt, _) = pair;
return new LiquidProjections.EventEnvelope
{
Body = evt,
Headers = evt
.Metadata
.Select(p => Tuple.Create <string, object>(p.Key, p.Value))
.ToDictionary(t => t.Item1, t => t.Item2)
};
}))
});
})
.Batch(5, t => ImmutableList <Transaction> .Empty.Add(t), (list, transaction) => list.Add(transaction))
.AlsoTo(Sink.OnComplete <ImmutableList <Transaction> >(
() => _isCancel.GetAndSet(true),
e =>
{
_isCancel.GetAndSet(true);
errorHandler(_exceptionInfo, e);
}))
.ViaMaterialized(KillSwitches.Single <ImmutableList <Transaction> >(), (_, kill) => kill)
.PreMaterialize(_materializer);
_cancelable = source.Item1;
var sinkQueue = source.Item2.RunWith(Sink.Queue <ImmutableList <Transaction> >()
.WithAttributes(new Attributes(new Attributes.InputBuffer(2, 2))), _materializer);
_runner = Run(sinkQueue);
return(this);
}
public void Cancel()
{
if (!_cancelled.GetAndSet(true))
{
_cancelCallback.Value?.Invoke(NotUsed.Instance);
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (stopped.GetAndSet(true))
{
// return if already stopped
return;
}
if (allocatorThread != null)
{
allocatorThread.Interrupt();
try
{
allocatorThread.Join();
}
catch (Exception ie)
{
Log.Warn("InterruptedException while stopping", ie);
}
}
if (isApplicationMasterRegistered && shouldUnregister)
{
Unregister();
}
base.ServiceStop();
}
public void SetOffline(bool offline)
{
_offline.GetAndSet(offline);
// Note that the offline state is known only to DefaultEventProcessor, not to EventDispatcher. We will
// simply avoid sending any flush messages to EventDispatcher if we're offline. EventDispatcher will
// never initiate a flush on its own.
}
/// <summary>
/// Kills the thread and releases all resources used by the
/// <see cref="ControlledRealTimeReopenThread{T}"/>. Also joins to the
/// thread so that when this method returns the thread is no longer alive.
/// </summary>
// LUCENENET specific - Support for Dispose(bool) since this is a non-sealed class.
protected virtual void Dispose(bool disposing)
{
// LUCENENET: Prevent double-dispose of our managed resources.
if (isDisposed.GetAndSet(true))
{
return;
}
if (disposing)
{
finish = true;
// So thread wakes up and notices it should finish:
reopenCond.Set();
try
{
Join();
}
catch (Exception ie) when(ie.IsInterruptedException())
{
throw new Util.ThreadInterruptedException(ie);
}
finally
{
RefreshDone();
// LUCENENET specific: dispose reset events
reopenCond.Dispose();
m_notify.Dispose();
}
}
}
/**
* Reset our local state: All values are set to 0.
*/
public void reset()
{
statsComputed.GetAndSet(false);
count = 0;
total = 0L;
mean = 0.0;
variance = 0.0;
stddev = 0.0;
min = 0L;
max = 0L;
sumSquares = 0.0;
for (int i = 0; i < percentileValues.Length; ++i)
{
percentileValues[i] = 0.0;
}
}
private void MaybeLogStoreError(Exception e)
{
if (!_loggedStorageError.GetAndSet(true))
{
LogHelpers.LogException(_log, "Failure in persistent data store", e);
}
}
public void TestGetAndSet()
{
var b1 = new AtomicBoolean(true);
bool result = b1.GetAndSet(false);
Assert.True(result);
Assert.False(b1.Get());
}
public void TestGetAndSet()
{
var b1 = new AtomicBoolean(true);
bool result = b1.GetAndSet(false);
Assert.Equal(true, result);
Assert.Equal(false, b1.Get());
}
private async Task UpdateTaskAsync()
{
try
{
var response = await _featureFlagRequestor.FeatureFlagsAsync();
if (response.statusCode == 200)
{
var flagsAsJsonString = response.jsonResponse;
var allData = DataModelSerialization.DeserializeV1Schema(flagsAsJsonString);
_updateSink.Init(_user, allData);
if (_initialized.GetAndSet(true) == false)
{
_startTask.SetResult(true);
_log.Info("Initialized LaunchDarkly Polling Processor.");
}
}
}
catch (UnsuccessfulResponseException ex)
{
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode);
if (HttpErrors.IsRecoverable(ex.StatusCode))
{
_log.Warn(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry"));
_updateSink.UpdateStatus(DataSourceState.Interrupted, errorInfo);
}
else
{
_log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", ""));
_updateSink.UpdateStatus(DataSourceState.Shutdown, errorInfo);
// if client is initializing, make it stop waiting
_startTask.TrySetResult(false);
((IDisposable)this).Dispose();
}
}
catch (JsonReadException ex)
{
_log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex));
_updateSink.UpdateStatus(DataSourceState.Interrupted,
new DataSourceStatus.ErrorInfo
{
Kind = DataSourceStatus.ErrorKind.InvalidData,
Time = DateTime.Now
});
}
catch (Exception ex)
{
Exception realEx = (ex is AggregateException ae) ? ae.Flatten() : ex;
_log.Warn("Polling for feature flag updates failed: {0}", LogValues.ExceptionSummary(realEx));
_log.Debug(LogValues.ExceptionTrace(realEx));
_updateSink.UpdateStatus(DataSourceState.Interrupted,
DataSourceStatus.ErrorInfo.FromException(realEx));
}
}
/// <summary>
/// Closes this selector.
///
/// <para> If the selector has already been closed then this method returns
/// immediately. Otherwise it marks the selector as closed and then invokes
/// the <seealso cref="#implCloseSelector implCloseSelector"/> method in order to
/// complete the close operation. </para>
/// </summary>
/// <exception cref="IOException">
/// If an I/O error occurs </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public final void close() throws java.io.IOException
public sealed override void Close()
{
bool open = SelectorOpen.GetAndSet(false);
if (!open)
{
return;
}
ImplCloseSelector();
}
public void AtomicBooleanGetAndSetReturnsOriginalValueMultiThreaded()
{
const int expected = 1000;
ConcurrentCountedSet <bool> values = new ConcurrentCountedSet <bool>();
var b = new AtomicBoolean(true);
System.Threading.Tasks.Parallel.For(0, 1000, i => values.Add(b.GetAndSet(i % 2 == 0)));
Assert.AreEqual(expected, values[true] + values[false]);
}
internal void SetDisabled(bool disabled)
{
if (_disabled.GetAndSet(disabled) != disabled)
{
DisabledChanged?.Invoke(null, new DisabledChangedArgs(disabled));
// We are not using TaskExecutor to dispatch this event because the
// event handler, if any, was not provided by the application - it is
// always our own internal logic in the LaunchDarkly.InternalSdk
// events code, which doesn't do anything time-consuming. So we are
// not calling out to unknown code and it's safe to be synchronous.
}
}
internal void UpdateAvailability(bool available)
{
if (_lastAvailable.GetAndSet(available) == available)
{
return; // no change
}
var status = new DataStoreStatus
{
Available = available,
RefreshNeeded = available && _refreshOnRecovery
};
if (available)
{
_log.Warn("Persistent store is available again");
}
_statusUpdater(status);
// If the store has just become unavailable, start a poller to detect when it comes back.
// If it has become available, stop any polling we are currently doing.
lock (_pollerLock)
{
if (available)
{
_pollCanceller?.Cancel();
_pollCanceller = null;
}
else
{
_log.Warn("Detected persistent store unavailability; updates will be cached until it recovers");
if (_pollCanceller is null)
{
// Start polling until the store starts working again
_pollCanceller = _taskExecutor.StartRepeatingTask(
PollInterval,
PollInterval,
() =>
{
if (_statusPollFn())
{
UpdateAvailability(true);
}
return(Task.FromResult(true)); // return value doesn't matter here
}
);
}
}
}
}
public void AtomicBooleanGetAndSetReturnsOriginalValue()
{
const int expected = 1000;
ConcurrentCountedSet <bool> values = new ConcurrentCountedSet <bool>();
var b = new AtomicBoolean();
for (int i = 0; i < 1000; i++)
{
values.Add(b.GetAndSet(i % 2 == 0));
}
Assert.AreEqual(expected, values[true] + values[false]);
}
public void A_lazy_source_must_work_never_construct_the_source_when_there_was_no_demand()
{
this.AssertAllStagesStopped(() =>
{
var probe = this.CreateSubscriberProbe <int>();
var constructed = new AtomicBoolean();
Source.Lazily(() =>
{
constructed.GetAndSet(true);
return(Source.From(new[] { 1, 2, 3 }));
}).RunWith(Sink.FromSubscriber(probe), Materializer);
probe.Cancel();
constructed.Value.Should().BeFalse();
}, Materializer);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (isStopping.GetAndSet(true))
{
return;
}
try
{
base.ServiceStop();
DefaultMetricsSystem.Shutdown();
}
finally
{
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
StopRecoveryStore();
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (stopped.GetAndSet(true))
{
// return if already stopped
return;
}
// shutdown any containers that might be left running
ShutdownAllContainers();
if (eventHandlingThread != null)
{
eventHandlingThread.Interrupt();
}
if (launcherPool != null)
{
launcherPool.ShutdownNow();
}
base.ServiceStop();
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (!_stopped.GetAndSet(true))
{
_flushTimer?.Dispose();
_flushUsersTimer?.Dispose();
SubmitMessage(new EventProcessorInternal.FlushMessage());
var message = new EventProcessorInternal.ShutdownMessage();
SubmitMessage(message);
message.WaitForCompletion();
_processorInternal.Dispose();
_messageQueue.CompleteAdding();
_messageQueue.Dispose();
}
}
}
public override void draw(float[] viewMatrix, float[] projectionMatrix)
{
lock (this)
{
GLES20.GlBindBuffer(GLES20.GlArrayBuffer, mVertexVBO);
if (mUpdateVBO.GetAndSet(false))
{
if (mPointCloudBuffer != null)
{
mPointCloudBuffer.Position(0);
// Pass the info to the VBO
GLES20.GlBufferData(GLES20.GlArrayBuffer, mPointCloudBuffer.Capacity() * BYTES_PER_FLOAT, mPointCloudBuffer, GLES20.GlStaticDraw);
mPointCount = mPointCloudBuffer.Capacity() / 3;
float totalZ = 0;
for (int i = 0; i < mPointCloudBuffer.Capacity() - 3; i = i + 3)
{
totalZ = totalZ + mPointCloudBuffer.Get(i + 2);
}
if (mPointCount != 0)
{
mAverageZ = totalZ / mPointCount;
}
// signal the update
mUpdateVBO.Set(true);
}
mPointCloudBuffer = null;
}
if (mPointCount > 0)
{
GLES20.GlUseProgram(mProgram);
updateMvpMatrix(viewMatrix, projectionMatrix);
GLES20.GlVertexAttribPointer(mPosHandle, COORDS_PER_VERTEX, GLES20.GlFloat, false, 0, 0);
GLES20.GlEnableVertexAttribArray(mPosHandle);
GLES20.GlUniformMatrix4fv(mMVPMatrixHandle, 1, false, MvpMatrix, 0);
GLES20.GlDrawArrays(GLES20.GlPoints, 0, mPointCount);
}
GLES20.GlBindBuffer(GLES20.GlArrayBuffer, 0);
}
}
private void SetupDiagnosticInit(bool enabled)
{
lock (_diagnosticTimerLock)
{
_diagnosticTimer?.Dispose();
_diagnosticTimer = null;
if (enabled)
{
TimeSpan initialDelay = _diagnosticRecordingInterval - (DateTime.Now - _diagnosticStore.DataSince);
TimeSpan safeDelay =
(initialDelay < TimeSpan.Zero) ?
TimeSpan.Zero :
((initialDelay > _diagnosticRecordingInterval) ? _diagnosticRecordingInterval : initialDelay);
_diagnosticTimer = new Timer(DoDiagnosticSend, null, safeDelay, _diagnosticRecordingInterval);
}
}
// Send initial and persisted unsent event the first time diagnostics are started
if (enabled && !_sentInitialDiagnostics.GetAndSet(true))
{
var unsent = _diagnosticStore.PersistedUnsentEvent;
var init = _diagnosticStore.InitEvent;
if (unsent.HasValue || init.HasValue)
{
Task.Run(async() => // do these in a single task for test determinacy
{
if (unsent.HasValue)
{
await _processorInternal.SendDiagnosticEventAsync(unsent.Value);
}
if (init.HasValue)
{
await _processorInternal.SendDiagnosticEventAsync(init.Value);
}
});
}
}
}
public AddressContext(string clientId, ArtemisClientManagerConfig managerConfig, string httpUrl, string wsEndpointSuffix)
{
Preconditions.CheckArgument(!string.IsNullOrWhiteSpace(clientId), "clientId");
Preconditions.CheckArgument(managerConfig != null, "managerConfig");
_ttl = managerConfig.ConfigurationManager.GetProperty(clientId + ".address.context-ttl", 5 * 60 * 1000, 5 * 60 * 1000, 30 * 60 * 1000);
if (string.IsNullOrWhiteSpace(httpUrl))
{
_httpUrl = string.Empty;
_webSocketEndpoint = string.Empty;
}
else
{
string url = httpUrl.Trim('/');
_httpUrl = url;
string wsEndpointPrefix = _httpSchema.Replace(url, _wsPrefix);
if (!string.IsNullOrWhiteSpace(wsEndpointSuffix))
{
_webSocketEndpoint = wsEndpointPrefix + "/" + wsEndpointSuffix.Trim('/');
}
_available.GetAndSet(true);
}
}
private async Task UpdateTaskAsync()
{
_log.Info("Polling LaunchDarkly for feature flag updates");
try
{
var allData = await _featureRequestor.GetAllDataAsync();
if (allData is null)
{
// This means it was cached, and alreadyInited was true
_dataSourceUpdates.UpdateStatus(DataSourceState.Valid, null);
}
else
{
if (_dataSourceUpdates.Init(allData.Value))
{
_dataSourceUpdates.UpdateStatus(DataSourceState.Valid, null);
if (!_initialized.GetAndSet(true))
{
_initTask.SetResult(true);
_log.Info("First polling request successful");
}
}
}
}
catch (UnsuccessfulResponseException ex)
{
_log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry"));
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode);
if (HttpErrors.IsRecoverable(ex.StatusCode))
{
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
}
else
{
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
try
{
// if client is initializing, make it stop waiting
_initTask.SetResult(true);
}
catch (InvalidOperationException)
{
// the task was already set - nothing more to do
}
((IDisposable)this).Dispose();
}
}
catch (JsonReadException ex)
{
_log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex));
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
new DataSourceStatus.ErrorInfo
{
Kind = DataSourceStatus.ErrorKind.InvalidData,
Time = DateTime.Now
});
}
catch (Exception ex)
{
Exception realEx = (ex is AggregateException ae) ? ae.Flatten() : ex;
LogHelpers.LogException(_log, "Polling for feature flag updates failed", realEx);
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
DataSourceStatus.ErrorInfo.FromException(realEx));
}
}
public void SetNextCheckLeaseResult(bool value) => nextCheckLeaseResult.GetAndSet(value);
public bool GetAndResetApplyAllDeletes()
{
return(flushDeletes.GetAndSet(false));
}
void ThreadStart()
{
/* Allow scheduler to break fairness of deque ordering without
* breaking its semantic (the task can be executed twice but the
* second time it will return immediately
*/
if (executing.GetAndSet(true))
{
return;
}
// Disable CancellationToken direct cancellation
if (cancellationTokenRegistration != null)
{
cancellationTokenRegistration.Value.Dispose();
cancellationTokenRegistration = null;
}
// If Task are ran inline on the same thread we might trash these values
var saveCurrent = current.Get();
var saveScheduler = TaskScheduler.Current;
current.Set(this);
#if NET_4_5
TaskScheduler.Current = HasFlag(creationOptions, TaskCreationOptions.HideScheduler) ? TaskScheduler.Default : scheduler;
#else
TaskScheduler.Current = scheduler;
#endif
if (!cancellationToken.IsCancellationRequested)
{
status = TaskStatus.Running;
try
{
InnerInvoke();
}
catch (OperationCanceledException oce)
{
if (cancellationToken != CancellationToken.None && oce.CancellationToken == cancellationToken)
{
CancelReal();
}
else
{
HandleGenericException(oce);
}
}
catch (Exception e)
{
HandleGenericException(e);
}
}
else
{
CancelReal();
}
if (saveCurrent != null)
{
current.Set(saveCurrent);
}
if (saveScheduler != null)
{
TaskScheduler.Current = saveScheduler;
}
Finish();
}
public override object DetermineCurrentLookupKey()
{
return(count.IncrementAndGet() > 3 ? null : (bool?)lookupFlag.GetAndSet(!lookupFlag.Value));
}
public async Task CommittablePartitionedSource_Should_handle_exceptions_in_stream_without_commit_failures()
{
var partitionsCount = 3;
var topic = CreateTopic(1);
var group = CreateGroup(1);
var totalMessages = 100;
var exceptionTriggered = new AtomicBoolean(false);
var allTopicPartitions = Enumerable.Range(0, partitionsCount).Select(i => new TopicPartition(topic, i)).ToList();
var consumerSettings = CreateConsumerSettings <string>(group).WithStopTimeout(TimeSpan.FromSeconds(2));
var createdSubSources = new ConcurrentSet <TopicPartition>();
var commitFailures = new ConcurrentSet <(TopicPartition, Exception)>();
var control = KafkaConsumer.CommittablePartitionedSource(consumerSettings, Subscriptions.Topics(topic))
.GroupBy(partitionsCount, tuple => tuple.Item1)
.SelectAsync(6, tuple =>
{
var(topicPartition, source) = tuple;
createdSubSources.TryAdd(topicPartition);
return(source
.Log($"Subsource for partition #{topicPartition.Partition.Value}", m => m.Record.Value)
.SelectAsync(3, async message =>
{
// fail on first partition; otherwise delay slightly and emit
if (topicPartition.Partition.Value == 0)
{
Log.Debug($"Failing {topicPartition} source");
exceptionTriggered.GetAndSet(true);
throw new Exception("FAIL");
}
else
{
await Task.Delay(50);
}
return message;
})
.Log($"Subsource {topicPartition} pre commit")
.SelectAsync(1, async message =>
{
try
{
await message.CommitableOffset.Commit();
}
catch (Exception ex)
{
Log.Error("Commit failure: " + ex);
commitFailures.TryAdd((topicPartition, ex));
}
return message;
})
.Scan(0, (c, _) => c + 1)
.RunWith(Sink.Last <int>(), Materializer)
.ContinueWith(t =>
{
Log.Info($"sub-source for {topicPartition} completed: Received {t.Result} messages in total.");
return t.Result;
}));
})
void HandleMessage(string messageType, string messageData)
{
_log.Debug("Event '{0}': {1}", messageType, messageData);
switch (messageType)
{
case Constants.PUT:
{
var allData = DataModelSerialization.DeserializeV1Schema(messageData);
_updateSink.Init(_user, allData);
if (!_initialized.GetAndSet(true))
{
_initTask.SetResult(true);
}
break;
}
case Constants.PATCH:
{
try
{
var parsed = LdValue.Parse(messageData);
var flagkey = parsed.Get(Constants.KEY).AsString;
var featureFlag = DataModelSerialization.DeserializeFlag(messageData);
_updateSink.Upsert(_user, flagkey, featureFlag.ToItemDescriptor());
}
catch (Exception ex)
{
LogHelpers.LogException(_log, "Error parsing PATCH message", ex);
_log.Debug("Message data follows: {0}", messageData);
}
break;
}
case Constants.DELETE:
{
try
{
var parsed = LdValue.Parse(messageData);
int version = parsed.Get(Constants.VERSION).AsInt;
string flagKey = parsed.Get(Constants.KEY).AsString;
var deletedItem = new ItemDescriptor(version, null);
_updateSink.Upsert(_user, flagKey, deletedItem);
}
catch (Exception ex)
{
LogHelpers.LogException(_log, "Error parsing DELETE message", ex);
_log.Debug("Message data follows: {0}", messageData);
}
break;
}
case Constants.PING:
{
Task.Run(async() =>
{
try
{
var response = await _requestor.FeatureFlagsAsync();
var flagsAsJsonString = response.jsonResponse;
var allData = DataModelSerialization.DeserializeV1Schema(flagsAsJsonString);
_updateSink.Init(_user, allData);
if (!_initialized.GetAndSet(true))
{
_initTask.SetResult(true);
}
}
catch (Exception ex)
{
LogHelpers.LogException(_log, "Error in handling PING message", ex);
}
});
break;
}
default:
break;
}
}