/// <summary> /// Check if there are any playing tracks on a node that has not shown signs of life in too long. In that case its /// playing tracks will also be marked dead. /// </summary> /// <param name="terminate"> Whether to terminate without checking the threshold </param> public virtual void processHealthCheck(bool terminate) { lock (this) { if (playingTracks.Empty || (!terminate && lastAliveTime >= DateTimeHelperClass.CurrentUnixTimeMillis() - TRACK_KILL_THRESHOLD)) { return; } connectionState.set(ConnectionState.OFFLINE.id()); if (!terminate) { log.LogWarning("Bringing node {} offline since last response from it was {}ms ago.", nodeAddress, DateTimeHelperClass.CurrentUnixTimeMillis() - lastAliveTime); } // There may be some racing that manages to add a track after this, it will be dealt with on the next iteration foreach (long?executorId in new IList <RemoteAudioTrackExecutor>(playingTracks.Keys)) { RemoteAudioTrackExecutor executor = playingTracks.remove(executorId); if (executor != null) { abandonedTrackManager.add(executor); } } } }
/// <summary> /// Distributes any abandoned tracks between the specified nodes. Only online nodes which are not under too heavy load /// are used. The number of tracks that can be assigned to a node depends on the number of tracks it is already /// processing (track count can increase only by 1/15th on each call, or by 5). /// </summary> /// <param name="nodes"> Remote nodes to give abandoned tracks to. </param> public virtual void distribute(IList <RemoteNodeProcessor> nodes) { if (abandonedExecutors.Empty) { return; } IList <Adopter> adopters = findAdopters(nodes); AbandonedExecutor executor; long currentTime = DateTimeHelperClass.CurrentUnixTimeMillis(); int maximum = getMaximumAdoptions(adopters); int assigned = 0; while (assigned < maximum && (executor = abandonedExecutors.poll()) != null) { if (checkValidity(executor, currentTime)) { Adopter adopter = selectNextAdopter(adopters); log.debug("Node {} is adopting {}.", adopter.node.Address, executor.executor); adopter.node.startPlaying(executor.executor); assigned++; } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private boolean dispatchOneTick(HttpInterface httpInterface, TickBuilder tickBuilder) throws Exception private bool dispatchOneTick(HttpInterface httpInterface, TickBuilder tickBuilder) { bool success = false; HttpPost post = new HttpPost("http://" + nodeAddress + "/tick"); abandonedTrackManager.distribute(Collections.singletonList(this)); ByteArrayEntity entity = new ByteArrayEntity(buildRequestBody()); post.GetEntity() = entity; tickBuilder.requestSize = (int)entity.ContentLength; CloseableHttpResponse response = httpInterface.execute(post); try { tickBuilder.responseCode = response.StatusLine.StatusCode; if (tickBuilder.responseCode != 200) { throw new IOException("Returned an unexpected response code " + tickBuilder.responseCode); } if (connectionState.compareAndSet(ConnectionState.PENDING.id(), ConnectionState.ONLINE.id())) { log.LogInformation("Node {} came online.", nodeAddress); } else if (connectionState.Get() != ConnectionState.ONLINE.id()) { log.LogWarning("Node {} received successful response, but had already lost control of its tracks.", nodeAddress); return(false); } lastAliveTime = DateTimeHelperClass.CurrentUnixTimeMillis(); if (!handleResponseBody(response.Entity.Content, tickBuilder)) { return(false); } success = true; } finally { if (!success) { IOUtils.closeQuietly(response); } else { IOUtils.closeQuietly(response.Entity.Content); } } return(true); }
/// <summary> /// Adds a track executor to abandoned tracks. The abandoned track manager will take over managing its lifecycle and /// the caller should not use it any further. /// </summary> /// <param name="executor"> The executor to register as an abandoned track. </param> public virtual void add(RemoteAudioTrackExecutor executor) { if (abandonedExecutors.offer(new AbandonedExecutor(DateTimeHelperClass.CurrentUnixTimeMillis(), executor))) { log.debug("{} has been put up for adoption.", executor); } else { log.debug("{} has been discarded, adoption queue is full.", executor); executor.dispatchException(new FriendlyException("Cannot find a node to play the track on.", Severity.COMMON, null)); executor.stop(); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private boolean processOneTick(HttpInterface httpInterface, RingBufferMath timingAverage) throws Exception private bool processOneTick(HttpInterface httpInterface, RingBufferMath timingAverage) { TickBuilder tickBuilder = new TickBuilder(DateTimeHelperClass.CurrentUnixTimeMillis()); try { if (!dispatchOneTick(httpInterface, tickBuilder)) { return(false); } } finally { tickBuilder.endTime = DateTimeHelperClass.CurrentUnixTimeMillis(); recordTick(tickBuilder.build(), timingAverage); } long sleepDuration = System.Math.Max((tickBuilder.startTime + 500) - tickBuilder.endTime, 10); System.Threading.Thread.Sleep((int)sleepDuration); return(true); }
public void run() { if (closed || !threadRunning.compareAndSet(false, true)) { log.LogDebug("Not running node processor for {}, thread already active.", nodeAddress); return; } log.LogDebug("Trying to connect to node {}.", nodeAddress); connectionState.set(ConnectionState.PENDING.id()); try { using (HttpInterface httpInterface = httpInterfaceManager.Interface) { RingBufferMath timingAverage = new RingBufferMath(10, @in => Math.Pow(@in, 5.0), @out => Math.Pow(@out, 0.2)); while (processOneTick(httpInterface, timingAverage)) { aliveTickCounter = System.Math.Max(1, aliveTickCounter + 1); lastAliveTime = DateTimeHelperClass.CurrentUnixTimeMillis(); } } } catch (InterruptedException) { log.LogInformation("Node {} processing was stopped.", nodeAddress); System.Threading.Thread.CurrentThread.Interrupt(); } catch (IOException e) { if (aliveTickCounter > 0) { log.LogError("Node {} went offline with exception.", nodeAddress, e); } else { log.LogDebug("Retry, node {} is still offline.", nodeAddress); } } catch (System.Exception e) { log.LogError("Node {} appears offline due to unexpected exception.", nodeAddress, e); ExceptionTools.rethrowErrors(e); } finally { processHealthCheck(true); connectionState.set(ConnectionState.OFFLINE.id()); aliveTickCounter = System.Math.Min(-1, aliveTickCounter - 1); threadRunning.set(false); if (!closed) { long delay = ScheduleDelay; if (aliveTickCounter == -1) { log.LogInformation("Node {} loop ended, retry scheduled in {}.", nodeAddress, delay); } scheduledExecutor.schedule(this, delay, TimeUnit.MILLISECONDS); } else { log.LogInformation("Node {} loop ended, node was removed.", nodeAddress); } } }