public IJobQueue Register()
{
IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
//
// NOTE: Amazon InstanceID and AMI removed
//
using (TurbineModelContainer container = new TurbineModelContainer())
{
container.JobConsumers.AddObject(new JobConsumer()
{
Id = consumerId,
hostname = hostname,
AMI = "",
instance = "",
processId = Process.GetCurrentProcess().Id.ToString(),
status = "up",
keepalive = DateTime.UtcNow
}
);
container.SaveChanges();
}
queue = new DBJobQueue();
return(queue);
}
public IJobQueue Register(IConsumerRun run)
{
this.run = run;
IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext();
//Guid consumerId = ctx.Id;
//Guid consumerId = Guid.NewGuid();
String hostname = ctx.Hostname;
String appName = run.SupportedApplications.ElementAtOrDefault(0);
Debug.WriteLine(String.Format("Register({0}) as {1}, {2}", appName, run.ConsumerId, hostname), this.GetType().Name);
using (ProducerContext db = new ProducerContext())
{
// TODO: Registering as a single application is dubious.
// IF support multiple apps in single consumer need to expose that via the database ( update SCHEMA!! )
var app = db.Applications.Single(a => a.Name == appName);
var consumer = new Turbine.Data.Entities.JobConsumer
{
Application = app,
Id = run.ConsumerId,
hostname = hostname,
processId = System.Diagnostics.Process.GetCurrentProcess().Id,
keepalive = DateTime.UtcNow,
status = "up"
};
db.Consumers.Add(consumer);
db.SaveChanges();
}
queue = AppUtility.GetJobQueue(run);
//((DBJobQueue)queue).consumerId = consumerId;
return(queue);
}
public BatchConsumeMessageContext(
IConsumerContext consumer,
IReadOnlyCollection <IMessageContext> batchMessage)
{
this.ConsumerContext = consumer;
this.Message = new Message(null, batchMessage);
}
public async ValueTask Execute(IConsumerContext context)
{
using var scope = Provider.CreateScope();
var pipeline = scope.ServiceProvider.GetRequiredService(context.ConsumerDefinition.PipelineType) as IConsumerPipeline
?? throw new ArgumentException($"{context.ConsumerDefinition.PipelineType} does not implement {typeof(IConsumerPipeline).FullName}");
await pipeline.Execute(context);
}
public async ValueTask Execute(IConsumerContext context)
{
var message = Serializer.Deserialize <TMessage>(context.Message);
if (message is not null)
{
await Consumer.Consume(context, message);
}
}
public async ValueTask Execute(IConsumerContext context)
{
using var transaction = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled);
var transactionContext = new TransactionContext(context);
await Pipeline.Execute(transactionContext);
await transactionContext.PublishMessages();
transaction.Complete();
}
public MessageContext(
Message message,
IMessageHeaders headers,
IConsumerContext consumer,
IProducerContext producer)
{
this.Message = message;
this.Headers = headers ?? new MessageHeaders();
this.ConsumerContext = consumer;
this.ProducerContext = producer;
}
public async ValueTask Execute(IConsumerContext context)
{
try
{
await _pipeline.Execute(context);
}
catch (Exception e)
{
_logger.LogError(e, context.Message.ToString());
}
}
private void AuthorizeConsumer(Job obj)
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerGUID = ctx.Id;
Guid jobGUID = obj.Consumer.Id;
if (jobGUID != Guid.Empty && jobGUID != consumerGUID)
{
throw new IllegalAccessException(String.Format(
"job({0}) consumer access denied {1} != {2}", obj.Id, jobGUID, consumerGUID)
);
}
}
public void Error()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("Error as {0}, {1}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.Single <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
consumer.status = "error";
container.SaveChanges();
}
}
public void TestMethod1()
{
consumerCtx = Turbine.Consumer.AppUtility.GetConsumerContext();
//appCtx = Turbine.Consumer.AppUtility.GetAppContext();
//Assert.AreNotSame(consumerCtx, Turbine.Consumer.AppUtility.GetConsumerContext());
IUnityContainer container = Turbine.Consumer.AppUtility.container;
//container.RegisterInstance<IContext>(consumerCtx);
container.RegisterInstance <IConsumerContext>(consumerCtx);
Assert.AreSame(consumerCtx, Turbine.Consumer.AppUtility.GetConsumerContext());
}
public static IEnumerable <TMessage> GetMessages <TMessage>(this IBus bus)
{
using (IConsumerContext <TMessage> context = bus.CreateConsumerContext <TMessage>())
{
foreach (var messageContext in Enumerator.Enumerate(context.GetMessage))
{
if (messageContext.AcceptanceRequired)
{
messageContext.AcceptMessage();
}
yield return(messageContext.Message);
}
}
}
public async ValueTask Execute(IConsumerContext context)
{
var retryContext = new ConsumerRetryContext(context);
await _retryPolicy.ExecuteAsync(async() =>
{
try
{
await _pipeline.Execute(retryContext);
}
catch (Exception)
{
retryContext.Increment();
throw;
}
});
}
public AzureConsumerContext(IConsumerContext consumerContext, ProcessMessageEventArgs azureContext, IServiceBus bus, ServiceBusSender sender)
{
_consumerContext = consumerContext;
_azureContext = azureContext;
_bus = bus;
_sender = sender;
if (_azureContext.Message.ApplicationProperties.TryGetValue(nameof(Redeliveries), out var raw))
{
if (raw is object && int.TryParse(raw.ToString(), out var i))
{
Redeliveries += i + 1;
}
}
Redeliveries += _consumerContext.Redeliveries;
}
public void Error()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("Error as {0}, {1}", consumerId), this.GetType().Name);
/*
* using (TurbineModelContainer container = new TurbineModelContainer())
* {
* JobConsumer consumer = (JobConsumer)container.Consumers.First<Turbine.Data.Consumer>(s => s.guid == consumerId);
* consumer.status = "error";
* container.SaveChanges();
* }
* */
}
public void KeepAlive()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("KeepAlive Consumer {0}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.Single <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
if (consumer.status == "up")
{
consumer.keepalive = DateTime.UtcNow;
}
container.SaveChanges();
}
}
/* UnRegister
* If state is "up" move to "down", else leave alone.
*/
public void UnRegister()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Debug.WriteLine(String.Format("UnRegister Consumer {0}", consumerId), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = (JobConsumer)container.JobConsumers.First <Turbine.Data.JobConsumer>(s => s.Id == consumerId);
if (consumer.status == "up")
{
consumer.status = "down";
}
container.SaveChanges();
}
}
public async ValueTask Execute(IConsumerContext context)
{
try
{
await Pipeline.Execute(context);
}
catch (Exception)
{
if (context.Redeliveries >= MaxRedeliveries)
{
throw;
}
var delay = DelayProvider(context.Redeliveries);
var redeliveryDate = DateTimeOffset.Now.Add(delay);
await context.Reschedule(redeliveryDate);
}
}
/* Setup: Moves job from submit to setup, concurrency issues to be aware of.
* Consumer must be registered before calling this method.
*
* throws OptimisticConcurrencyException if job has already been
* grabbed by another consumer.
*
*/
public IProcess Setup()
{
string baseDir = AppUtility.GetAppContext().BaseWorkingDirectory;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
Guid pid = Guid.Empty;
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = container.JobConsumers
.SingleOrDefault <Turbine.Data.JobConsumer>(c => c.Id == consumerId);
Debug.WriteLine(String.Format("Setup consumer GUID {0}", consumerId), this.GetType().Name);
if (consumer == null)
{
throw new IllegalAccessException("Setup failed, Consumer is not registered");
}
if (consumer.status != "up")
{
throw new IllegalAccessException(String.Format(
"Setup failed, Consumer status {0} != up", consumer.status));
}
Job obj = container.Jobs.Single <Job>(s => s.Id == id);
if (obj.State != "locked")
{
throw new InvalidStateChangeException("Violation of state machine");
}
if (obj.Process == null)
{
throw new InvalidStateChangeException("Process configuration missing");
}
obj.State = "setup";
obj.Setup = DateTime.UtcNow;
obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
obj.Consumer = (JobConsumer)consumer;
pid = obj.Process.Id;
simulationName = obj.Simulation.Name;
container.SaveChanges();
}
return(SinterProcessContract.Get(pid));
}
public IJobQueue Register()
{
IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
Turbine.Consumer.AWS.IAWSContext awsCtx = null;
string ami = "";
string instance = "";
try
{
awsCtx = Turbine.Consumer.AWS.AppUtility.GetContext();
ami = awsCtx.AmiId;
instance = awsCtx.InstanceId;
}
catch (Exception ex)
{
Debug.WriteLine(String.Format("Ignore AWS Context is not available: {0}", ex.Message), this.GetType().Name);
Debug.WriteLine(String.Format("{0}", ex.StackTrace), this.GetType().Name);
}
//
// NOTE: Amazon InstanceID and AMI removed
//
using (TurbineModelContainer container = new TurbineModelContainer())
{
container.JobConsumers.AddObject(new JobConsumer()
{
Id = consumerId,
hostname = hostname,
AMI = ami,
instance = instance,
processId = Process.GetCurrentProcess().Id.ToString(),
status = "up"
}
);
container.SaveChanges();
}
queue = new Turbine.Consumer.Data.Contract.DBJobQueue();
return(queue);
}
public void Register()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
using (NHibernate.ISession session = SessionFactory.OpenSession())
{
using (NHibernate.ITransaction trans = session.BeginTransaction())
{
var consumer = new JobConsumer()
{
Id = consumerId, hostname = hostname, status = "up"
};
}
}
}
public void Register()
{
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
using (TurbineModelContainer container = new TurbineModelContainer())
{
container.Consumers.AddObject(new JobConsumer()
{
guid = consumerId,
hostname = hostname,
status = "up"
}
);
container.SaveChanges();
}
}
static void Main(string[] args)
{
var options = new Options();
string dir = Properties.Settings.Default.BaseDirectory;
IUnityContainer container = Turbine.Consumer.AppUtility.container;
IConsumerContext consumerCtx = Turbine.Consumer.AppUtility.GetConsumerContext();
container.RegisterInstance <IConsumerContext>(consumerCtx);
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
// TODO: HACK TO BIND SIMULATION NAME
consumerCtx.BindSimulationName = options.Simulation;
}
int iterations = Properties.Settings.Default.TimeOutIterations;
int setupIterations = Properties.Settings.Default.TimeOutSetupIterations;
int postInitIterations = Properties.Settings.Default.TimePostInitIterations;
Turbine.Consumer.Console.SinterConsumerConsole.setTimeOutParams(iterations,
setupIterations, postInitIterations);
Turbine.Consumer.Console.SinterConsumerConsole.Run();
}
/* Setup: Moves job from submit to setup, concurrency issues to be aware of.
* Consumer must be registered before calling this method.
*
* throws OptimisticConcurrencyException if job has already been
* grabbed by another consumer.
*
*/
public IProcess Setup()
{
string baseDir = AppUtility.GetAppContext().BaseWorkingDirectory;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
using (TurbineModelContainer container = new TurbineModelContainer())
{
Consumer consumer = container.Consumers
.SingleOrDefault <Consumer>(c => c.guid == consumerId);
if (consumer == null)
{
throw new IllegalAccessException("Setup failed, Consumer is not registered");
}
Job obj = container.Jobs.Single <Job>(s => s.Id == id);
if (obj.State != "submit")
{
throw new InvalidStateChangeException("Violation of state machine");
}
if (obj.Process == null)
{
throw new InvalidStateChangeException("Process configuration missing");
}
obj.State = "setup";
obj.Setup = DateTime.UtcNow;
obj.Process.WorkingDir = Path.Combine(baseDir, id.ToString());
obj.JobConsumer = (JobConsumer)consumer;
container.SaveChanges();
}
return(SinterProcessContract.Get(id));
}
// Consumer grabs job in submit state off queue
public IJobConsumerContract GetNext()
{
IJobConsumerContract contract = new AspenJobConsumerContract();;
IConsumerContext ctx = AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
using (TurbineModelContainer container = new TurbineModelContainer())
{
var consumer = container.JobConsumers
.SingleOrDefault <Turbine.Data.JobConsumer>(c => c.Id == consumerId);
if (consumer == null)
{
throw new IllegalAccessException("GetNext failed, Consumer is not registered");
}
if (consumer.status != "up")
{
throw new IllegalAccessException(String.Format(
"GetNext failed, Consumer status {0} != up", consumer.status));
}
//Job obj = container.Jobs.OrderBy("it.Submit").
// FirstOrDefault<Job>(s => s.State == "submit");
//obj.Simulation.ApplicationName
Job obj = null;
// TODO: CHange the WHERE clause to do an OR between all available applications
if (applications == null)
{
System.Data.Objects.ObjectQuery <Turbine.Data.Job> query = container.Jobs;
obj = query.OrderBy("it.Submit").FirstOrDefault <Job>(s => s.State == "submit");
}
else
{
foreach (string appName in applications)
{
System.Data.Objects.ObjectQuery <Turbine.Data.Job> query = container.Jobs;
query = query.Where("it.Simulation.ApplicationName = @appName",
new System.Data.Objects.ObjectParameter("appName", appName));
obj = query.OrderBy("it.Submit").FirstOrDefault <Job>(s => s.State == "submit");
if (obj != null)
{
break;
}
}
}
if (obj == null)
{
return(null);
}
// Need to Check that Consumer is up
obj.State = "locked";
try
{
container.SaveChanges();
}
catch (System.Data.OptimisticConcurrencyException)
{
Debug.WriteLine("OptimisticConcurrencyException: Failed attempt to lock job", GetType().Name);
return(null);
}
contract.Id = obj.Id;
}
return(contract);
}
public async ValueTask Execute(IConsumerContext context)
{
using (await _limiter.WaitAsync())
await _pipeline.Execute(context);
}
public JsonPlaceHolderConsumer(IConsumerContext <JsonPlaceHolderConsumer> context)
: base(context)
{
}
protected RxConsumer(IConsumerContext <RxConsumer> context)
{
this.context = context;
http = new RxHttpClient(context.Http, context.Logger);
}
public static IConsumerContext GetConsumerContext()
{
IConsumerContext ctx = container.Resolve <IConsumerContext>();
return(ctx);
}
public IJobConsumerContract GetNext(IConsumerRun run)
{
apps = run.SupportedApplications;
IJobConsumerContract contract = null;
if (apps.Length == 0)
{
throw new JobQueueException("Invalid JobQueue configuration: no applications specified");
}
IConsumerContext appContext = AppUtility.GetConsumerContext();
string simulationName = appContext.BindSimulationName;
using (ProducerContext db = new ProducerContext())
{
Job job = null;
foreach (var appName in apps)
{
Debug.WriteLine("Get Job for Application: " + appName, "DBJobQueue.GetNext");
if (String.IsNullOrEmpty(simulationName))
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == run.ConsumerId &&
j.Simulation.ApplicationName == appName).FirstOrDefault();
if (job == null)
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == null &&
j.Simulation.ApplicationName == appName).FirstOrDefault();
}
}
else
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == run.ConsumerId &&
j.Simulation.Name == simulationName && j.Simulation.ApplicationName == appName).FirstOrDefault();
if (job == null)
{
job = db.Jobs.OrderByDescending(j => j.Submit).
Where(j => j.State == "submit" && j.ConsumerId == null &&
j.Simulation.Name == simulationName && j.Simulation.ApplicationName == appName).FirstOrDefault();
}
}
var consumer = db.Consumers.Single(c => c.Id == run.ConsumerId);
if (job != null)
{
job.State = "locked";
job.Consumer = consumer;
db.SaveChanges();
contract = new ConsumerJobContract(job.Count, run.ConsumerId);
Debug.WriteLine(String.Format("Found Job({0}): {1},{2}", job.Id, job.Simulation.Name, job.Simulation.ApplicationName), "DBJobQueue.GetNext");
break;
}
}
}
return(contract);
}
