/// <summary>
/// Check if there are any playing tracks on a node that has not shown signs of life in too long. In that case its
/// playing tracks will also be marked dead.
/// </summary>
/// <param name="terminate"> Whether to terminate without checking the threshold </param>
public virtual void processHealthCheck(bool terminate)
{
lock (this)
{
if (playingTracks.Empty || (!terminate && lastAliveTime >= DateTimeHelperClass.CurrentUnixTimeMillis() - TRACK_KILL_THRESHOLD))
{
return;
}
connectionState.set(ConnectionState.OFFLINE.id());
if (!terminate)
{
log.LogWarning("Bringing node {} offline since last response from it was {}ms ago.", nodeAddress, DateTimeHelperClass.CurrentUnixTimeMillis() - lastAliveTime);
}
// There may be some racing that manages to add a track after this, it will be dealt with on the next iteration
foreach (long?executorId in new IList <RemoteAudioTrackExecutor>(playingTracks.Keys))
{
RemoteAudioTrackExecutor executor = playingTracks.remove(executorId);
if (executor != null)
{
abandonedTrackManager.add(executor);
}
}
}
}
/// <summary>
/// Distributes any abandoned tracks between the specified nodes. Only online nodes which are not under too heavy load
/// are used. The number of tracks that can be assigned to a node depends on the number of tracks it is already
/// processing (track count can increase only by 1/15th on each call, or by 5).
/// </summary>
/// <param name="nodes"> Remote nodes to give abandoned tracks to. </param>
public virtual void distribute(IList <RemoteNodeProcessor> nodes)
{
if (abandonedExecutors.Empty)
{
return;
}
IList <Adopter> adopters = findAdopters(nodes);
AbandonedExecutor executor;
long currentTime = DateTimeHelperClass.CurrentUnixTimeMillis();
int maximum = getMaximumAdoptions(adopters);
int assigned = 0;
while (assigned < maximum && (executor = abandonedExecutors.poll()) != null)
{
if (checkValidity(executor, currentTime))
{
Adopter adopter = selectNextAdopter(adopters);
log.debug("Node {} is adopting {}.", adopter.node.Address, executor.executor);
adopter.node.startPlaying(executor.executor);
assigned++;
}
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private boolean dispatchOneTick(HttpInterface httpInterface, TickBuilder tickBuilder) throws Exception
private bool dispatchOneTick(HttpInterface httpInterface, TickBuilder tickBuilder)
{
bool success = false;
HttpPost post = new HttpPost("http://" + nodeAddress + "/tick");
abandonedTrackManager.distribute(Collections.singletonList(this));
ByteArrayEntity entity = new ByteArrayEntity(buildRequestBody());
post.GetEntity() = entity;
tickBuilder.requestSize = (int)entity.ContentLength;
CloseableHttpResponse response = httpInterface.execute(post);
try
{
tickBuilder.responseCode = response.StatusLine.StatusCode;
if (tickBuilder.responseCode != 200)
{
throw new IOException("Returned an unexpected response code " + tickBuilder.responseCode);
}
if (connectionState.compareAndSet(ConnectionState.PENDING.id(), ConnectionState.ONLINE.id()))
{
log.LogInformation("Node {} came online.", nodeAddress);
}
else if (connectionState.Get() != ConnectionState.ONLINE.id())
{
log.LogWarning("Node {} received successful response, but had already lost control of its tracks.", nodeAddress);
return(false);
}
lastAliveTime = DateTimeHelperClass.CurrentUnixTimeMillis();
if (!handleResponseBody(response.Entity.Content, tickBuilder))
{
return(false);
}
success = true;
}
finally
{
if (!success)
{
IOUtils.closeQuietly(response);
}
else
{
IOUtils.closeQuietly(response.Entity.Content);
}
}
return(true);
}
/// <summary>
/// Adds a track executor to abandoned tracks. The abandoned track manager will take over managing its lifecycle and
/// the caller should not use it any further.
/// </summary>
/// <param name="executor"> The executor to register as an abandoned track. </param>
public virtual void add(RemoteAudioTrackExecutor executor)
{
if (abandonedExecutors.offer(new AbandonedExecutor(DateTimeHelperClass.CurrentUnixTimeMillis(), executor)))
{
log.debug("{} has been put up for adoption.", executor);
}
else
{
log.debug("{} has been discarded, adoption queue is full.", executor);
executor.dispatchException(new FriendlyException("Cannot find a node to play the track on.", Severity.COMMON, null));
executor.stop();
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private boolean processOneTick(HttpInterface httpInterface, RingBufferMath timingAverage) throws Exception
private bool processOneTick(HttpInterface httpInterface, RingBufferMath timingAverage)
{
TickBuilder tickBuilder = new TickBuilder(DateTimeHelperClass.CurrentUnixTimeMillis());
try
{
if (!dispatchOneTick(httpInterface, tickBuilder))
{
return(false);
}
}
finally
{
tickBuilder.endTime = DateTimeHelperClass.CurrentUnixTimeMillis();
recordTick(tickBuilder.build(), timingAverage);
}
long sleepDuration = System.Math.Max((tickBuilder.startTime + 500) - tickBuilder.endTime, 10);
System.Threading.Thread.Sleep((int)sleepDuration);
return(true);
}
public void run()
{
if (closed || !threadRunning.compareAndSet(false, true))
{
log.LogDebug("Not running node processor for {}, thread already active.", nodeAddress);
return;
}
log.LogDebug("Trying to connect to node {}.", nodeAddress);
connectionState.set(ConnectionState.PENDING.id());
try
{
using (HttpInterface httpInterface = httpInterfaceManager.Interface)
{
RingBufferMath timingAverage = new RingBufferMath(10, @in => Math.Pow(@in, 5.0), @out => Math.Pow(@out, 0.2));
while (processOneTick(httpInterface, timingAverage))
{
aliveTickCounter = System.Math.Max(1, aliveTickCounter + 1);
lastAliveTime = DateTimeHelperClass.CurrentUnixTimeMillis();
}
}
}
catch (InterruptedException)
{
log.LogInformation("Node {} processing was stopped.", nodeAddress);
System.Threading.Thread.CurrentThread.Interrupt();
}
catch (IOException e)
{
if (aliveTickCounter > 0)
{
log.LogError("Node {} went offline with exception.", nodeAddress, e);
}
else
{
log.LogDebug("Retry, node {} is still offline.", nodeAddress);
}
}
catch (System.Exception e)
{
log.LogError("Node {} appears offline due to unexpected exception.", nodeAddress, e);
ExceptionTools.rethrowErrors(e);
}
finally
{
processHealthCheck(true);
connectionState.set(ConnectionState.OFFLINE.id());
aliveTickCounter = System.Math.Min(-1, aliveTickCounter - 1);
threadRunning.set(false);
if (!closed)
{
long delay = ScheduleDelay;
if (aliveTickCounter == -1)
{
log.LogInformation("Node {} loop ended, retry scheduled in {}.", nodeAddress, delay);
}
scheduledExecutor.schedule(this, delay, TimeUnit.MILLISECONDS);
}
else
{
log.LogInformation("Node {} loop ended, node was removed.", nodeAddress);
}
}
}
