public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting cluster...");
_appLifetime.ApplicationStarted.Register(() => SafeTask.Run(RunRequestLoop, _appLifetime.ApplicationStopping));
_appLifetime.ApplicationStopping.Register(OnStopping);
return(Task.CompletedTask);
}
public void SafeContinueWith_NothingSet_HandleExceptionRuns()
{
SpecificException specificException = new SpecificException();
SafeExecutionHelpers.RevertToDefaultImplementation();
var mockHelpers = new Mock <ISafeExecutionHelpers>();
SafeExecutionHelpers.Implementation = mockHelpers.Object;
var dts = new DeterministicTaskScheduler(shouldThrowExceptions: false);
var sut = new SafeTask();
sut.SafeContinueWith <Exception>(
Task.Factory.StartNew(
() => throw specificException,
CancellationToken.None,
TaskCreationOptions.None,
dts),
null,
dts);
dts.RunTasksUntilIdle();
Assert.Contains(specificException, dts.Exceptions);
mockHelpers.Verify(h => h.HandleException <Exception>(specificException, null));
SafeExecutionHelpers.RevertToDefaultImplementation();
}
public Task ReceiveAsync(IContext context)
{
switch (context.Message)
{
case Started _:
Console.WriteLine("LoopActor - Started");
context.Send(context.Self, new LoopParentMessage());
break;
case LoopParentMessage _:
_ = SafeTask.Run(async() => {
context.Send(context.Parent, new RenameCommand {
Name = GeneratePronounceableName(5)
});
await Task.Delay(TimeSpan.FromMilliseconds(500));
context.Send(context.Self, new LoopParentMessage());
}
);
break;
}
return(Task.CompletedTask);
}
public void SafeContinueWith_OnExceptionExceptionSet_HandlesException()
{
SpecificException specificException = new SpecificException();
Action <Exception> onException = new Mock <Action <Exception> >().Object;
SafeExecutionHelpers.Initialize();
var mockHelpers = new Mock <ISafeExecutionHelpers>();
SafeExecutionHelpers.SetImplementation(mockHelpers.Object);
var dts = new DeterministicTaskScheduler(shouldThrowExceptions: false);
var sut = new SafeTask();
sut.SafeContinueWith <Exception>(
Task.Factory.StartNew(
() => throw specificException,
CancellationToken.None,
TaskCreationOptions.None,
dts),
onException, //The crux
dts);
dts.RunTasksUntilIdle();
Assert.Contains(specificException, dts.Exceptions);
mockHelpers.Verify(h => h.HandleException(specificException, onException));
SafeExecutionHelpers.RevertToDefaultImplementation();
}
public void SafeContinueWith_DefaultExceptionHandlerSet_HandlesException()
{
SpecificException specificException = new SpecificException();
Action <Exception> defaultExceptionHandler = new Mock <Action <Exception> >().Object;
SafeExecutionHelpers.Initialize();
var mockHelpers = new Mock <ISafeExecutionHelpers>();
//Crux - DefaultHandler returns non-null delegate
mockHelpers.SetupGet(h => h.DefaultExceptionHandler).Returns(defaultExceptionHandler);
SafeExecutionHelpers.SetImplementation(mockHelpers.Object);
var dts = new DeterministicTaskScheduler(shouldThrowExceptions: false);
var sut = new SafeTask();
sut.SafeContinueWith <Exception>(
Task.Factory.StartNew(
() => throw specificException,
CancellationToken.None,
TaskCreationOptions.None,
dts),
null,
dts);
dts.RunTasksUntilIdle();
Assert.Contains(specificException, dts.Exceptions);
mockHelpers.Verify(h => h.HandleException <Exception>(specificException, null));
SafeExecutionHelpers.RevertToDefaultImplementation();
}
/// <summary>Creates a new, empty file in the directory with the given name.
/// Returns a stream writing this file.
/// </summary>
public override IndexOutput CreateOutput(string name, IOContext context)
{
var loc = GetLocation(name);
var streamToken = new TaskCompletionSource <IWriteAsyncStream>();
var completeToken = new TaskCompletionSource <bool>();
var saveTask = Task.Run(() => CreateSaveTask(loc, streamToken, completeToken.Task));
var stream = SafeTask.SafeResult(() => streamToken.Task);
return(new SecureStoreIndexOutput(name, stream, () =>
{
SafeTask.SafeWait(async() =>
{
completeToken.SetResult(true);
var m = await saveTask.ConfigureAwait(false);
// We didn't know the content length until now, so save it as a final step
var metadata = new Metadata();
metadata.ContentLength = m.ContentLength.Value;
await _store.SaveMetadata(loc, metadata, _options).ConfigureAwait(false);
});
}));
}
public override bool Obtain()
{
if (_lock == null)
{
_lock = SafeTask.SafeResult(() => _store.Lock(_location));
}
return(_lock != null);
}
/// <summary>Removes an existing file in the directory. </summary>
public override void DeleteFile(string name)
{
var location = GetLocation(name);
SafeTask.SafeWait(() => _store.Delete(location, null, _options));
LeoTrace.WriteLine(String.Format("DELETE {0}", location.BasePath));
_memoryCache.Remove(GetCacheKey(name));
}
public Task StartAsync()
{
var props = Props.FromProducer(() => new ClusterHeartBeatActor());
_pid = _context.SpawnNamed(props, ClusterHeartBeatName);
_logger = Log.CreateLogger("ClusterHeartBeat-" + _cluster.LoggerId);
_logger.LogInformation("Started Cluster Heartbeats");
_ = SafeTask.Run(HeartBeatLoop);
return(Task.CompletedTask);
}
protected override void FlushBuffer(byte[] b, int offset, int len)
{
if (_hasDisposed)
{
throw new ObjectDisposedException(nameof(SecureStoreIndexOutput));
}
_length += len;
SafeTask.SafeWait(() => _writeStream.WriteAsync(b, offset, len, CancellationToken.None));
}
/// <summary>Returns the length of a file in the directory. </summary>
public override long FileLength(string name)
{
if (_memoryCache.TryGetValue <byte[]>(GetCacheKey(name), out var bytes))
{
return(bytes.LongLength);
}
var metadata = SafeTask.SafeResult(() => _store.GetMetadata(GetLocation(name)));
return(metadata == null || !metadata.ContentLength.HasValue ? 0 : metadata.ContentLength.Value);
}
public override bool FileExists(string name)
{
if (_memoryCache.TryGetValue <byte[]>(GetCacheKey(name), out var bytes))
{
return(true);
}
var metadata = SafeTask.SafeResult(() => _store.GetMetadata(GetLocation(name)));
return(metadata != null);
}
public async Task <bool> Run()
{
_cluster = await CreateCluster().ConfigureAwait(false);
if (_mainWorker != null)
{
this._mainWorkerTask = SafeTask.Run(() => _mainWorker.Run(_cluster), _cancellationTokenSource.Token);
}
return(true);
}
public void Dispose()
{
if (_uploader != null)
{
// We just need to finish this off on a new thread
try
{
SafeTask.SafeWait(() => _uploader.Abort());
} catch (Exception) { }
_uploader = null;
}
_isComplete = true;
}
/// <summary>Returns a stream reading an existing file. </summary>
public override IndexInput OpenInput(string name, IOContext context)
{
var data = SafeTask.SafeResult(() => _memoryCache.GetOrCreateAsync(GetCacheKey(name), async e =>
{
e.SetSlidingExpiration(TimeSpan.FromHours(1)).SetPriority(CacheItemPriority.Low);
var loc = GetLocation(name);
var enc = await _encryptor.Value.ConfigureAwait(false);
var stream = await _store.LoadData(loc, null, enc).ConfigureAwait(false);
if (stream == null)
{
throw new System.IO.FileNotFoundException(name);
}
return(await stream.Stream.ReadBytes().ConfigureAwait(false));
}));
return(new SecureStoreIndexInput(data));
}
public void Wait(Func <Task> producer)
{
//block caller
_semaphore.Wait();
_ = SafeTask.Run(async() => {
try
{
var task = producer();
await task;
}
finally
{
//release once the async flow is done
_semaphore.Release();
}
}
);
}
public SecureStoreDirectory(ISecureStore store, string container, string basePath, Lazy <Task <IEncryptor> > encryptor, IMemoryCache memoryCache = null, string cachePrefix = null)
{
_container = container;
_memoryCache = memoryCache ?? new MemoryCache(new MemoryCacheOptions());
_cachePrefix = cachePrefix ?? "Lucene";
_basePath = basePath ?? string.Empty;
_store = store;
_encryptor = encryptor ?? new Lazy <Task <IEncryptor> >(() => Task.FromResult((IEncryptor)null));
_lockFactory = new SecureLockFactory(store, GetLocation);
_options = SecureStoreOptions.None;
if (_store.CanCompress)
{
_options = _options | SecureStoreOptions.Compress;
}
SafeTask.SafeWait(() => store.CreateContainerIfNotExists(container));
}
public void Start()
{
CurrentBucket = Task.Delay(_bucketSize, _ct);
_ = SafeTask.Run(async() => {
while (!_ct.IsCancellationRequested)
{
try
{
CurrentBucket = Task.Delay(_bucketSize, _ct);
await Task.Delay(_updateInterval, _ct);
}
catch (OperationCanceledException)
{
//pass, expected
}
}
}
);
}
/// <summary>
/// This has no guarantees that the throttle opens exactly after the period, since it is reset asynchronously
/// Throughput has been prioritized over exact re-opening
/// </summary>
/// <param name="maxEventsInPeriod"></param>
/// <param name="period"></param>
/// <param name="throttledCallBack">This will be called with the number of events what was throttled after the period</param>
/// <returns></returns>
public static ShouldThrottle Create(
int maxEventsInPeriod,
TimeSpan period,
Action <int>?throttledCallBack = null
)
{
if (maxEventsInPeriod == 0)
{
return(() => Valve.Closed);
}
if (period == TimeSpan.Zero || maxEventsInPeriod < 1 || maxEventsInPeriod == int.MaxValue)
{
return(() => Valve.Open);
}
var currentEvents = 0;
return(() => {
var tries = Interlocked.Increment(ref currentEvents);
if (tries == 1)
{
StartTimer(throttledCallBack);
}
if (tries == maxEventsInPeriod)
{
return Valve.Closing;
}
return tries > maxEventsInPeriod ? Valve.Closed : Valve.Open;
});
void StartTimer(Action <int>?callBack) => _ = SafeTask.Run(async() => {
await Task.Delay(period);
var timesCalled = Interlocked.Exchange(ref currentEvents, 0);
if (timesCalled > maxEventsInPeriod)
{
callBack?.Invoke(timesCalled - maxEventsInPeriod);
}
}
);
}
private void StartUpdateTtlLoop() => _ = SafeTask.Run(async() => {
while (!_shutdown)
{
try
{
await _client.Agent.PassTTL("service:" + _consulServiceInstanceId, "");
await Task.Delay(_refreshTtl, _cluster.System.Shutdown);
}
catch (Exception x)
{
if (!_cluster.System.Shutdown.IsCancellationRequested)
{
_logger.LogError(x, "Consul TTL Loop failed");
}
}
}
_logger.LogInformation("Consul Exiting TTL loop");
}
);
private Task RestartMe()
{
_ = SafeTask.Run(async() =>
{
await Task.Delay(2000);
try{ await this.Shutdown(); }
catch
{
// ignored
}
_cluster = null;
await Task.Delay(5000);
_cancellationTokenSource = new CancellationTokenSource();
await this.Run();
await Task.Delay(2000);
});
return(Task.CompletedTask);
}
public override int Read(byte[] buffer, int offset, int count)
{
int read = 0;
int currentCount = count;
while (currentCount > 0)
{
// We are completely done
if (_isDone && _bufferNeedsData && _bufferNeedsData2)
{
return(read);
}
// We ran out of read data...
if (!_isDone && _bufferNeedsData && _bufferNeedsData2)
{
// If we have read something, just return that for now
if (read > 0)
{
return(read);
}
// Otherwise we are forced to load more data :(
// Note: this could cause deadlock? Shouldn't in Leo Engine...
SafeTask.SafeWait(() => PrepareData());
continue;
}
if (_isFirstBuffer)
{
if (_bufferNeedsData)
{
_isFirstBuffer = false;
}
else
{
var toRead = Math.Min(currentCount, _length - _offset);
Buffer.BlockCopy(_buffer, _offset, buffer, offset, toRead);
offset += toRead;
_offset += toRead;
read += toRead;
currentCount -= toRead;
if (_offset == _length)
{
_bufferNeedsData = true;
_isFirstBuffer = _bufferNeedsData2; // Only flip buffers if we actually have data in the other buffer
}
}
}
else
{
if (_bufferNeedsData2)
{
_isFirstBuffer = true;
}
else
{
var toRead = Math.Min(currentCount, _length2 - _offset2);
Buffer.BlockCopy(_buffer2, _offset2, buffer, offset, toRead);
offset += toRead;
_offset2 += toRead;
read += toRead;
currentCount -= toRead;
if (_offset2 == _length2)
{
_bufferNeedsData2 = true;
_isFirstBuffer = true;
}
}
}
}
return(read);
}
public async Task InitializeDb(DbConnectionString cnxStr)
{
_dbConnectionString = cnxStr;
DbDisplayName = _dbConnectionString.DisplayName;
await SafeTask.RunSafe(() => InitializeDbAsync(), HandleError);
}
/// <summary>Returns an array of strings, one for each file in the directory. </summary>
public override string[] ListAll()
{
return(SafeTask.SafeResult(() => ListAllAsync()));
}
private static async Task Main()
{
Log.SetLoggerFactory(LoggerFactory.Create(c => c
.SetMinimumLevel(LogLevel.Information)
.AddConsole()
)
);
var logger = Log.CreateLogger <Program>();
#if NETCORE
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
#endif
Console.WriteLine("Enter 0 to use GrpcCore provider");
Console.WriteLine("Enter 1 to use GrpcNet provider");
if (!int.TryParse(Console.ReadLine(), out var provider))
{
provider = 0;
}
Console.WriteLine("Enter client advertised host (Enter = localhost)");
var advertisedHost = Console.ReadLine().Trim();
if (string.IsNullOrEmpty(advertisedHost))
{
advertisedHost = "127.0.0.1";
}
Console.WriteLine("Enter remote advertised host (Enter = localhost)");
var remoteAddress = Console.ReadLine().Trim();
if (string.IsNullOrEmpty(remoteAddress))
{
remoteAddress = "127.0.0.1";
}
var actorSystemConfig = new ActorSystemConfig()
.WithDeadLetterThrottleCount(10)
.WithDeadLetterThrottleInterval(TimeSpan.FromSeconds(2));
var system = new ActorSystem(actorSystemConfig);
var context = new RootContext(system);
IRemote remote;
if (provider == 0)
{
var remoteConfig = GrpcCoreRemoteConfig
.BindTo(advertisedHost)
.WithProtoMessages(ProtosReflection.Descriptor);
remote = new GrpcCoreRemote(system, remoteConfig);
}
else
{
var remoteConfig = GrpcNetRemoteConfig
.BindTo(advertisedHost)
.WithChannelOptions(new GrpcChannelOptions
{
CompressionProviders = new[]
{
new GzipCompressionProvider(CompressionLevel.Fastest)
}
}
)
.WithProtoMessages(ProtosReflection.Descriptor);
remote = new GrpcNetRemote(system, remoteConfig);
}
await remote.StartAsync();
var messageCount = 1000000;
var cancellationTokenSource = new CancellationTokenSource();
_ = SafeTask.Run(async() => {
while (!cancellationTokenSource.IsCancellationRequested)
{
var semaphore = new SemaphoreSlim(0);
var props = Props.FromProducer(() => new LocalActor(0, messageCount, semaphore));
var pid = context.Spawn(props);
try
{
var actorPidResponse =
await remote.SpawnAsync($"{remoteAddress}:12000", "echo", TimeSpan.FromSeconds(1));
if (actorPidResponse.StatusCode == (int)ResponseStatusCode.OK)
{
var remotePid = actorPidResponse.Pid;
await context.RequestAsync <Start>(remotePid, new StartRemote {
Sender = pid
},
TimeSpan.FromSeconds(1)
);
var stopWatch = new Stopwatch();
stopWatch.Start();
Console.WriteLine("Starting to send");
var msg = new Ping();
for (var i = 0; i < messageCount; i++)
{
context.Send(remotePid, msg);
}
var linkedTokenSource =
CancellationTokenSource.CreateLinkedTokenSource(cancellationTokenSource.Token,
new CancellationTokenSource(2000).Token
);
await semaphore.WaitAsync(linkedTokenSource.Token);
stopWatch.Stop();
var elapsed = stopWatch.Elapsed;
Console.WriteLine("Elapsed {0}", elapsed);
var t = messageCount * 2.0 / elapsed.TotalMilliseconds * 1000;
Console.Clear();
Console.WriteLine("Throughput {0} msg / sec", t);
await context.StopAsync(remotePid);
}
}
catch (OperationCanceledException)
{
await Task.Delay(1000);
}
catch (Exception e)
{
logger?.LogError(e, "Error");
await Task.Delay(5000);
}
await context.PoisonAsync(pid);
}
}, cancellationTokenSource.Token
);
Console.ReadLine();
cancellationTokenSource.Cancel();
await Task.Delay(1000);
Console.WriteLine("Press enter to quit");
Console.ReadLine();
await remote.ShutdownAsync();
}
public async Task <Cluster> CreateCluster()
{
try
{
var actorSystemConfig = ActorSystemConfig.Setup();
if (_metricsProvider != null)
{
actorSystemConfig = actorSystemConfig
.WithMetricsProviders(_metricsProvider);
}
var system = new ActorSystem(actorSystemConfig);
_logger.LogInformation("Setting up Cluster");
_logger.LogInformation("ClusterName: " + _clusterSettings.ClusterName);
_logger.LogInformation("PIDDatabaseName: " + _clusterSettings.PIDDatabaseName);
_logger.LogInformation("PIDCollectionName: " + _clusterSettings.PIDCollectionName);
var clusterProvider = _clusterProvider.CreateClusterProvider(_logger);
//var identity = RedisIdentityLookup.GetIdentityLookup(_clusterSettings.ClusterName, _clusterSettings.Host, _clusterSettings.RedisPort);
var identity = MongoIdentityLookup.GetIdentityLookup(_clusterSettings.ClusterName, _clusterSettings.PIDConnectionString, _clusterSettings.PIDCollectionName, _clusterSettings.PIDDatabaseName);
var(clusterConfig, remoteConfig) = GenericClusterConfig.CreateClusterConfig(_clusterSettings, clusterProvider, identity, _descriptorProvider, _logger);
if (_setupRootActors != null)
{
clusterConfig = _setupRootActors.AddRootActors(clusterConfig);
}
_ = new GrpcCoreRemote(system, remoteConfig);
var cluster = new Cluster(system, clusterConfig);
await cluster.StartMemberAsync().ConfigureAwait(false);
if (this._subscriptionFactory != null)
{
_logger.LogInformation("Fire up subscriptions for system {id} {address}", system.Id, system.Address);
await this._subscriptionFactory.FireUp(system).ConfigureAwait(false);
}
_ = SafeTask.Run(async() =>
{
try
{
int counter = 0;
while (!_cancellationTokenSource.IsCancellationRequested)
{
Member[] members = cluster.MemberList.GetAllMembers();
string[] clusterKinds = cluster.GetClusterKinds();
if (clusterKinds.Length == 0)
{
_logger.LogInformation("[SharedClusterWorker] clusterKinds {clusterKinds}", clusterKinds.Length);
_logger.LogInformation("[SharedClusterWorker] Restarting");
_ = this.RestartMe();
break;
}
this.Connected = members.Length > 0;
if (!this.Connected)
{
counter = 0;
_logger.LogInformation("[SharedClusterWorker] Connected {Connected}", this.Connected);
}
if (this.Connected)
{
if (counter % 20 == 0)
{
_logger.LogDebug("[SharedClusterWorker] Members {@Members}", members.Select(m => m.ToLogString()));
}
counter++;
}
await Task.Delay(500);
}
}
catch
{
// ignored
}
}, _cancellationTokenSource.Token);
return(cluster);
}
catch (Exception ex)
{
_logger.LogError(ex, "SharedClusterWork failed");
throw;
}
}
public static void Run()
{
var l = LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Information));
Log.SetLoggerFactory(l);
var config = ActorSystemConfig.Setup().WithMetricsProviders(new PrometheusConfigurator());
var remoteConfig = GrpcCoreRemoteConfig
.BindToLocalhost()
.WithProtoMessages(MessagesReflection.Descriptor);
var clusterConfig =
ClusterConfig
.Setup("MyCluster", new ConsulProvider(new ConsulProviderConfig(), c => c.Address = new Uri("http://127.0.0.1:8500/")),
new PartitionIdentityLookup()
);
var system = new ActorSystem(config)
.WithRemote(remoteConfig)
.WithCluster(clusterConfig);
system
.Cluster()
.StartMemberAsync();
var props = Props.FromProducer(() => new MyActor());
var config2 = ActorSystemConfig.Setup().WithMetricsProviders(new PrometheusConfigurator());
var remoteConfig2 = GrpcCoreRemoteConfig
.BindToLocalhost()
.WithProtoMessages(MessagesReflection.Descriptor);
var clusterConfig2 =
ClusterConfig
.Setup("MyCluster", new ConsulProvider(new ConsulProviderConfig(), c => c.Address = new Uri("http://127.0.0.1:8500/")),
new PartitionIdentityLookup()
)
.WithClusterKind("somekind", props);
var system2 = new ActorSystem(config2)
.WithRemote(remoteConfig2)
.WithCluster(clusterConfig2);
system2
.Cluster()
.StartMemberAsync();
_ = SafeTask.Run(async() => {
var r = new Random();
while (true)
{
await Task.Delay(r.Next(1, 2000));
await system.Cluster().RequestAsync <SomeResponse>($"someactor{r.Next(1, 100)}", "somekind", new SomeRequest(),
CancellationTokens.WithTimeout(5000)
);
}
}
);
}
private void StartMonitorMemberStatusChangesLoop()
{
_ = SafeTask.Run(async() => {
var waitIndex = 0ul;
while (!_shutdown && !_cluster.System.Shutdown.IsCancellationRequested)
{
try
{
var statuses = await _client.Health.Service(_consulServiceName, null, false, new QueryOptions
{
WaitIndex = waitIndex,
WaitTime = _blockingWaitTime
}
, _cluster.System.Shutdown
);
if (_deregistered)
{
break;
}
_logger.LogDebug("Got status updates from Consul");
waitIndex = statuses.LastIndex;
var currentMembers =
statuses
.Response
.Where(v => IsAlive(v.Checks)) //only include members that are alive
.Select(ToMember)
.ToArray();
_memberList.UpdateClusterTopology(currentMembers);
}
catch (Exception x)
{
if (!_cluster.System.Shutdown.IsCancellationRequested)
{
_logger.LogError(x, "Consul Monitor failed");
//just backoff and try again
await Task.Delay(2000);
}
}
}
}
);
Member ToMember(ServiceEntry v)
{
var member = new Member
{
Id = v.Service.Meta["id"],
Host = v.Service.Address,
Port = v.Service.Port
};
member.Kinds.AddRange(v.Service.Tags);
return(member);
}
}
/// <summary>
/// Allows <code>await AsyncPropertyBase</code> rather than requiring <code>await AsyncPropertyBase.SafeTask</code>.
/// </summary>
/// <returns>The <see cref="TaskAwaiter"/> from <see cref="IAsyncProperty{T}.SafeTask"/></returns>
public TaskAwaiter GetAwaiter() => SafeTask.GetAwaiter();
