public Follower(IZooKeeperService zooKeeperService,
IRebalancerLogger logger,
ResourceManager store,
string clientId,
int clientNumber,
string watchSiblingPath,
TimeSpan sessionTimeout,
TimeSpan onStartDelay,
CancellationToken followerToken)
{
this.zooKeeperService = zooKeeperService;
this.logger = logger;
this.store = store;
this.clientId = clientId;
this.clientNumber = clientNumber;
this.watchSiblingPath = watchSiblingPath;
siblingId = watchSiblingPath.Substring(watchSiblingPath.LastIndexOf("/", StringComparison.Ordinal));
this.sessionTimeout = sessionTimeout;
this.onStartDelay = onStartDelay;
this.followerToken = followerToken;
rebalancingCts = new CancellationTokenSource();
events = new BlockingCollection <FollowerEvent>();
disconnectedTimer = new Stopwatch();
}
public async Task StartAsync(IRebalancerLogger logger)
{
CreateNewClient(logger);
await Client.StartAsync(ResourceGroup, ClientOptions);
Started = true;
}
public Follower(IRebalancerLogger logger,
IClientService clientService,
ResourceGroupStore store)
{
this.logger = logger;
this.clientService = clientService;
this.store = store;
}
private void CreateNewClient(IRebalancerLogger logger)
{
Id = $"Client{ClientNumber}";
ClientNumber++;
Monitor.RegisterAddClient(Id);
Client = new RebalancerClient();
Client.OnAssignment += (sender, args) =>
{
Resources = args.Resources;
foreach (string resource in args.Resources)
{
Monitor.ClaimResource(resource, Id);
}
if (onStartTime > TimeSpan.Zero)
{
if (randomiseTimes)
{
double waitTime = onStartTime.TotalMilliseconds * rand.NextDouble();
Thread.Sleep((int)waitTime);
}
else
{
Thread.Sleep(onStartTime);
}
}
};
Client.OnUnassignment += (sender, args) =>
{
foreach (string resource in Resources)
{
Monitor.ReleaseResource(resource, Id);
}
Resources.Clear();
if (onStopTime > TimeSpan.Zero)
{
if (randomiseTimes)
{
double waitTime = onStopTime.TotalMilliseconds * rand.NextDouble();
Thread.Sleep((int)waitTime);
}
else
{
Thread.Sleep(onStopTime);
}
}
};
Client.OnAborted += (sender, args) =>
{
logger.Info("CLIENT", $"CLIENT ABORTED: {args.AbortReason}");
};
}
public Coordinator(IRebalancerLogger logger,
IResourceService resourceService,
IClientService clientService,
ResourceGroupStore store)
{
this.logger = logger;
this.resourceService = resourceService;
this.clientService = clientService;
this.store = store;
resources = new List <string>();
clients = new List <Guid>();
}
public async Task PerformActionAsync(IRebalancerLogger logger)
{
if (Started)
{
logger.Info("TEST RUNNER", "Stopping client");
Monitor.RegisterRemoveClient(Id);
await StopAsync();
logger.Info("TEST RUNNER", "Stopped client");
}
else
{
logger.Info("TEST RUNNER", "Starting client");
await StartAsync(logger);
logger.Info("TEST RUNNER", "Started client");
}
}
public Coordinator(IZooKeeperService zooKeeperService,
IRebalancerLogger logger,
ResourceManager store,
string clientId,
TimeSpan minimumRebalancingInterval,
TimeSpan sessionTimeout,
TimeSpan onStartDelay,
CancellationToken coordinatorToken)
{
this.zooKeeperService = zooKeeperService;
this.logger = logger;
this.store = store;
this.minimumRebalancingInterval = minimumRebalancingInterval;
this.clientId = clientId;
this.sessionTimeout = sessionTimeout;
this.onStartDelay = onStartDelay;
this.coordinatorToken = coordinatorToken;
rebalancingCts = new CancellationTokenSource();
events = new BlockingCollection <CoordinatorEvent>();
disconnectedTimer = new Stopwatch();
}
public LeaseService(string connectionString, IRebalancerLogger logger)
{
this.connectionString = connectionString;
this.logger = logger;
}
public async Task TryPutResourceBarrierAsync(string resource, CancellationToken waitToken,
IRebalancerLogger logger)
{
Stopwatch sw = new();
sw.Start();
string actionToPerform = $"try put resource barrier on {resource}";
bool succeeded = false;
while (!succeeded)
{
await BlockUntilConnected(actionToPerform);
try
{
await zookeeper.createAsync(
$"{resourcesPath}/{resource}/barrier",
Encoding.UTF8.GetBytes(clientId),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL);
succeeded = true;
}
catch (KeeperException.NodeExistsException)
{
(bool exists, string owner) = await GetResourceBarrierOwnerAsync(resource);
if (exists && owner.Equals(clientId))
{
succeeded = true;
}
else
{
logger.Info(clientId, $"Waiting for {owner} to release its barrier on {resource}");
// wait for two seconds, will retry in next iteration
for (int i = 0; i < 20; i++)
{
await WaitFor(TimeSpan.FromMilliseconds(100));
if (waitToken.IsCancellationRequested)
{
throw new ZkOperationCancelledException(
$"Could not {actionToPerform} as the operation was cancelled.");
}
}
}
}
catch (KeeperException.NoNodeException e)
{
throw new ZkInvalidOperationException(
$"Could not {actionToPerform} as the resource node does not exist.", e);
}
catch (KeeperException.ConnectionLossException)
{
// do nothing, the next iteration will try again
}
catch (KeeperException.SessionExpiredException e)
{
throw new ZkSessionExpiredException($"Could not {actionToPerform} as the session has expired: ", e);
}
catch (Exception e)
{
throw new ZkInvalidOperationException($"Could not {actionToPerform} due to an unexpected error", e);
}
}
}
