public async void Add_Job_To_Queue()
{
_jobQueue.Enqueue(CreateTestJob());
var result = await _jobManager.GetJobStatus(_jobId);
Assert.AreEqual(JobStatus.Completed, result);
}
public async Task <Guid> CreateTask(CancellationToken token)
{
var taskId = Guid.NewGuid();
var newTask = new Job(taskId);
await _repository.Add(newTask, token);
_logger.LogInformation($"Task {taskId} was created");
_queue.Enqueue(newTask);
return(taskId);
}
public void CreateJob(JobType jobType, Stream jobStream)
{
var worker = workers.FirstOrDefault(worker => worker.JobType == jobType);
if (worker == null)
{
throw new ArgumentOutOfRangeException(nameof(jobType), jobType, $"No worker setup to execute {jobType.GetDisplayName()}");
}
jobQueue.Enqueue(worker.Execute(jobStream));
}
public void FillQueue(IJobQueue queue)
{
var submissionInfos = new WaitingSubmissionsSortedByTimeAsc()
.Load(session)
.List <Submission>()
.Select(x => new SubmissionInfo(x));
foreach (var submissionInfo in submissionInfos)
{
queue.Enqueue(submissionInfo, Priority.Normal);
}
}
internal void AcceptProposals(IConfirmation jobConfirmation)
{
Agent.DebugMessage($"Start Acknowledge Job {jobConfirmation.Key} {jobConfirmation.Job.Name} | scope : [{jobConfirmation.ScopeConfirmation.GetScopeStart()} to {jobConfirmation.ScopeConfirmation.GetScopeEnd()}]" +
$" with Priority {jobConfirmation.Job.Priority(Agent.CurrentTime)}" +
$" | scopeLimit: {_scopeQueue.Limit} | scope workload : {_scopeQueue.Workload} | Capacity left {_scopeQueue.Limit - _scopeQueue.Workload} " +
$" {((FBucket)jobConfirmation.Job).MaxBucketSize} ", CustomLogger.PRIORITY, LogLevel.Warn);
var setup = jobConfirmation.CapabilityProvider.ResourceSetups.Single(x =>
x.Resource.IResourceRef != null &&
((IActorRef)x.Resource.IResourceRef).Path.Name == Agent.Context.Self.Path.Name);
var isQueueAble = _scopeQueue.CheckScope(jobConfirmation, Agent.CurrentTime, _jobInProgress.ResourceIsBusyUntil, _capabilityProviderManager.GetCurrentUsedCapabilityId(), setup.UsedInSetup);
if (isQueueAble && _jobInProgress.IsSet &&
_jobInProgress.ResourceIsBusyUntil > jobConfirmation.ScopeConfirmation.GetScopeStart())
{
Agent.DebugMessage(msg: $"Seems to be wrong #3", CustomLogger.JOB, LogLevel.Warn);
}
// If is not queueable
if (!isQueueAble)
{
Agent.DebugMessage(msg: $"Reject proposal for: {jobConfirmation.Job.Name} {jobConfirmation.Key} with jobPrio: { jobConfirmation.Job.Priority(Agent.CurrentTime) } and send reject job to job agent", CustomLogger.PRIORITY, LogLevel.Warn);
Agent.Send(instruction: Job.Instruction.StartRequeue.Create(target: jobConfirmation.JobAgentRef));
return;
}
this.UpdateAndRequeuePlanedJobs(jobConfirmation);
_scopeQueue.Enqueue(jobConfirmation);
Agent.DebugMessage(msg: $"Accepted proposal on resource {Agent.Context.Self.Path.Name} and " +
$"start enqueue {jobConfirmation.Job.Name} {jobConfirmation.Key} queueCount: {_scopeQueue.Count}" +
$"", CustomLogger.PRIORITY, LogLevel.Warn);
UpdateProcessingItem();
}
public TPLDataflowWithErrorHandling(IJobQueue <string> poisonQueue)
{
var policy =
Policy.Handle <Exception>() // on any exception
.Retry(3); // retry 3 times
_jobs = new ActionBlock <Action>((job) =>
{
try
{
policy.Execute(() =>
{
int customer = GetCustomerById(job);// possibly throws exception
//Console.WriteLine(customer.Name);
});
}
catch (Exception e)
{
// If policy failed (after 3 retries), move to poison queue
poisonQueue.Enqueue(job);
}
});
}
public async Task HandleCallbackQueryAsync(CallbackQuery callbackQuery)
{
if (callbackQuery?.Data != null)
{
switch (callbackQuery.Data)
{
case var s when _menu.Select(x => x.RunName).Contains(callbackQuery.Data):
{
var result = true;
var menuItem = _menu.FirstOrDefault(x => x.RunName == callbackQuery.Data);
try
{
await _jenkinsService.RunJobAsync(menuItem.Path, menuItem.Parameters.ToDictionary(x => x.Name, y => y.Value));
}
catch (Exception ex)
{
result = false;
Log.Error(ex, "Run handler");
}
try
{
var message = result ? $"{menuItem.DisplayName} has been enqueued successfully" : $"Failed to enqueue {menuItem.DisplayName}";
await _client.SendTextMessageAsync(callbackQuery.Message.Chat.Id, message);
}
catch (Exception ex)
{
Log.Error(ex, "Run handler. Send text.");
}
if (result && menuItem.NotifyWhenBuildIsFinished)
{
_jobQueue.Enqueue(new JobSettings {
ChatId = callbackQuery.Message.Chat.Id, JobPath = menuItem.Path, JobDisplayName = menuItem.DisplayName
});
}
}
break;
case var s when _menu.Select(x => x.StatusName).Contains(callbackQuery.Data):
{
var status = (JobStatus)null;
var menuItem = _menu.FirstOrDefault(x => x.StatusName == callbackQuery.Data);
try
{
status = await _jenkinsService.GetJobStatusAsync(menuItem.Path);
}
catch (Exception ex)
{
Log.Error(ex, "Status handler");
}
try
{
var message = status != null ? $"{menuItem.DisplayName} is {status.Status} on {status.TimeStamp.FromUnixTimeMilliseconds():dd.MM.yyyy HH:mm}" : $"Failed to get a status of the job";
await _client.SendTextMessageAsync(callbackQuery.Message.Chat.Id, message);
}
catch (Exception ex)
{
Log.Error(ex, "Status handler. Send text.");
}
}
break;
case var s when _menu.Select(x => x.ArtifactName).Contains(callbackQuery.Data):
{
var result = true;
var menuItem = _menu.FirstOrDefault(x => x.ArtifactName == callbackQuery.Data);
try
{
var artifacts = await _jenkinsService.GetJobArtifactsAsync(menuItem.Path);
foreach (var artifact in artifacts)
{
await _client.SendDocumentAsync(callbackQuery.Message.Chat.Id, new InputOnlineFile(artifact.Item2, artifact.Item1));
}
artifacts.ToList().ForEach(x => x.Item2.Dispose());
}
catch (Exception ex)
{
result = false;
Log.Error(ex, "Artifact handler");
}
try
{
if (!result)
{
await _client.SendTextMessageAsync(callbackQuery.Message.Chat.Id, $"Failed to download artifacts");
}
}
catch (Exception ex)
{
Log.Error(ex, "Artifact handler. Send text.");
}
}
break;
}
}
}
public void Unicast(UdpClientRemoteInfo remoteInfo, MemoryStream[] frames, Action action)
{
// Frames larger than half the maximum packet size certainly cannot be packed together.
var smallFrames = new List <MemoryStream>(frames.Length);
var largeFrames = new List <MemoryStream>(frames.Length);
const int kHalfMaximumTransportSize = UdpConstants.kMaximumTransportSize / 2;
for (var i = 0; i < frames.Length; i++)
{
var frame = frames[i];
if (frame.Length <= kHalfMaximumTransportSize)
{
smallFrames.Add(frame);
}
else
{
largeFrames.Add(frame);
}
}
// Order small frames ascending by size, large frames descending by size.
smallFrames.Sort(new MemoryStreamByPositionComparer());
largeFrames.Sort(new ReverseComparer <MemoryStream>(new MemoryStreamByPositionComparer()));
// Place large frames into outbound buffers.
var outboundBuffers = new List <MemoryStream>(frames.Length);
foreach (var largeFrame in largeFrames)
{
var outboundBuffer = outboundMemoryStreamPool.TakeObject();
outboundBuffer.Write(largeFrame.GetBuffer(), 0, (int)largeFrame.Position);
outboundBuffers.Add(outboundBuffer);
}
// Place small frames into outbound buffers. Note that as the
// small frames are ascending in size and the buffers are descending
// in size, while we iterate if a small frame cannot fit into the
// next outbound buffer then none of the following small frames can either.
int activeOutboundBufferIndex = 0;
foreach (var smallFrame in smallFrames)
{
// precompute greatest outbound buffer permission for which
// we will still be able to fit into the buffer.
int frameSize = (int)smallFrame.Position;
int greatestFittableBufferPosition = UdpConstants.kMaximumTransportSize - frameSize;
// Attempt to place the small frame into existing outbound buffers
bool placed = false;
while (!placed && activeOutboundBufferIndex != outboundBuffers.Count)
{
var outboundBuffer = outboundBuffers[activeOutboundBufferIndex];
if (outboundBuffer.Position > greatestFittableBufferPosition)
{
activeOutboundBufferIndex++;
}
else
{
outboundBuffer.Write(smallFrame.GetBuffer(), 0, (int)smallFrame.Position);
placed = true;
}
}
// If no existing outbound buffer had space, allocate a new one
if (!placed)
{
AssertEquals(outboundBuffers.Count, activeOutboundBufferIndex);
var outboundBuffer = outboundMemoryStreamPool.TakeObject();
outboundBuffer.Write(smallFrame.GetBuffer(), 0, (int)smallFrame.Position);
outboundBuffers.Add(outboundBuffer);
}
}
// Console.WriteLine($"Batched {frames.Length} to {outboundBuffers.Count} buffers.");
int sendsRemaining = outboundBuffers.Count;
foreach (var outboundBuffer in outboundBuffers)
{
var job = new UdpUnicastJob {
OutboundBuffer = outboundBuffer,
RemoteInfo = remoteInfo,
SendCompletionHandler = () => {
outboundBuffer.SetLength(0);
outboundMemoryStreamPool.ReturnObject(outboundBuffer);
if (Interlocked.Decrement(ref sendsRemaining) == 0)
{
action();
}
}
};
unicastJobQueue.Enqueue(job);
}
// int sendsRemaining = outboundBuffers.Count;
// Parallel.ForEach(
// outboundBuffers,
// outboundBuffer => {
// outboundBytesAggregator.Put(outboundBuffer.Length);
//
// var e = new SocketAsyncEventArgs();
// e.RemoteEndPoint = remoteInfo.IPEndpoint;
// e.SetBuffer(outboundBuffer.GetBuffer(), 0, (int)outboundBuffer.Position);
// e.Completed += (sender, args) => {
// // Duplicate code with below.
// args.SetBuffer(null, 0, 0);
// args.Dispose();
//
// outboundBuffer.SetLength(0);
// outboundMemoryStreamPool.ReturnObject(outboundBuffer);
//
// if (Interlocked.Decrement(ref sendsRemaining) == 0) {
// action();
// }
// };
//
// const int kSendStateAsync = 1;
// const int kSendStateDone = 2;
// const int kSendStateError = 3;
// int sendState;
// try {
// bool completingAsynchronously = remoteInfo.Socket.SendToAsync(e);
// sendState = completingAsynchronously ? kSendStateAsync : kSendStateDone;
// } catch (ObjectDisposedException) when (isShutdown) {
// sendState = kSendStateError;
// }
//
// if (sendState == kSendStateDone || sendState == kSendStateError) {
// // Completed synchronously so e.Completed won't be called.
// e.SetBuffer(null, 0, 0);
// e.Dispose();
//
// outboundBuffer.SetLength(0);
// outboundMemoryStreamPool.ReturnObject(outboundBuffer);
//
// if (Interlocked.Decrement(ref sendsRemaining) == 0) {
// action();
// }
// }
// });
// int sendsRemaining = outboundBuffers.Count;
// foreach (var outboundBuffer in outboundBuffers) {
// outboundBytesAggregator.Put(outboundBuffer.Length);
//
// var e = new SocketAsyncEventArgs();
// e.RemoteEndPoint = remoteInfo.IPEndpoint;
// e.SetBuffer(outboundBuffer.GetBuffer(), 0, (int)outboundBuffer.Position);
// e.Completed += (sender, args) => {
// // Duplicate code with below.
// args.SetBuffer(null, 0, 0);
// args.Dispose();
//
// outboundBuffer.SetLength(0);
// outboundMemoryStreamPool.ReturnObject(outboundBuffer);
//
// if (Interlocked.Decrement(ref sendsRemaining) == 0) {
// action();
// }
// };
//
// const int kSendStateAsync = 1;
// const int kSendStateDone = 2;
// const int kSendStateError = 3;
// int sendState;
// try {
// bool completingAsynchronously = remoteInfo.Socket.SendToAsync(e);
// sendState = completingAsynchronously ? kSendStateAsync : kSendStateDone;
// } catch (ObjectDisposedException) when (isShutdown) {
// sendState = kSendStateError;
// }
//
// if (sendState == kSendStateDone || sendState == kSendStateError) {
// // Completed synchronously so e.Completed won't be called.
// e.SetBuffer(null, 0, 0);
// e.Dispose();
//
// outboundBuffer.SetLength(0);
// outboundMemoryStreamPool.ReturnObject(outboundBuffer);
//
// if (sendState == kSendStateError) {
// // Don't send remaining messages.
// // To the application, this appears like packet loss.
// action();
// return;
// } else if (Interlocked.Decrement(ref sendsRemaining) == 0) {
// action();
// }
// }
// }
}
public override void AddToQueue(string queue, string jobId) => _queue.Enqueue(queue, jobId);
private void DoOneJob(IJobQueue <Action> jobQueue)
{
jobQueue.Enqueue(() => _autoResetEvent.Set());
_autoResetEvent.WaitOne();
jobQueue.Stop();
}
