public JobExecutor(JobQueue jobQueue)
{
_status = JobExecutorStatus.Idle;
_shouldStop = false;
JobQueue = jobQueue;
JobQueue.ProgressDelegate = this;
}
public void InitQueue()
{
var fileRep = new FileQueueRepository<Job>(Environment.CurrentDirectory + "\\queue");
_fileQueue = new JobQueue { Repository = fileRep };
//_fileQueue.Clear();
}
public void StoppedJobShouldRemainInTheQueue()
{
_jobQueue = new JobQueue { ProgressDelegate = this };
_jobQueue.Enqueue(new JobToSucceed());
_jobQueue.Execute();
_jobQueue.Count.ShouldEqual(1, "Stopping after first job's first task execution should leave 1 job in the queue.");
}
public void SetupQueue()
{
JobConfiguration.AppSettings = Settings.Default;
Repository = new FileQueueRepository<Job>(Environment.CurrentDirectory + "\\queue");
var fileErrorRep = new FileQueueRepository<Job>(Environment.CurrentDirectory + "\\queue-error");
var fileExecutedRep = new FileQueueRepository<Job>(Environment.CurrentDirectory + "\\queue-executed");
JobQueue = new JobQueue { Repository = Repository, ErroredJobs = fileErrorRep, ExecutedJobs = fileExecutedRep, LoggerDelegate = Logger};
JobQueue.Clear();
JobQueue.ErroredJobs.Clear();
JobQueue.ExecutedJobs.Clear();
}
public void SetupQueue()
{
JobConfiguration.AppSettings = Settings.Default;
var conn = new SqlConnection { ConnectionString = Settings.Default.ConnectionString };
Repository = new SqlQueueRepository<Job>(conn, "ActiveItems");
var sqlErrorRep = new SqlQueueRepository<Job>(conn, "ErroredItems");
var sqlExecutedRep = new SqlQueueRepository<Job>(conn, "ExecutedItems");
JobQueue = new JobQueue { Repository = Repository, ErroredJobs = sqlErrorRep, ExecutedJobs = sqlExecutedRep, LoggerDelegate = Logger };
JobQueue.Clear();
JobQueue.ErroredJobs.Clear();
JobQueue.ExecutedJobs.Clear();
}
private static void Main(string[] args)
{
var app = new FirebaseApp(new Uri("https://dazzling-fire-1575.firebaseio.com/"),
"T7dtZlSSDnGFZkPVWGq9chNPZ6dJQyt4bQNl4udN");
JobQueue<Job> queue = new JobQueue<Job>(app.Child("jobs/job"), job =>
{
Console.WriteLine("[{0}]: {1}", job.Priority, job.Description);
return true;
});
var feederStart = new ParameterizedThreadStart(Feeder);
Thread feeder = new Thread(feederStart);
feeder.Start(queue);
Console.ReadLine();
}
public void TestDataSet(DataSource input, string output)
{
var assignment = new JobQueue();
Assert.Equal(output, assignment.Execute(input));
}
public ExcessWrites()
{
_excessItem = _world.NewJob;
_queue = _world.NewJobQueue(configuration: _configuration);
_queue.Write(_world.NewJob);
}
public async Task <int> AddJobQueue(int projectId, string originUrl, string jobType, int?jobDefinitionId, CancellationToken cancellationToken = default(CancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var project = await _projectRepository.GetById(projectId, cancellationToken);
bool projectAllowAddingQueue = project?.Status == ProjectStatusFilterType.Active ||
(project?.Status == ProjectStatusFilterType.Deleting && jobType == JobType.Delete);
if (project == null || !projectAllowAddingQueue)
{
throw new ProjectNotFoundException(projectId);
}
var inProgressJobSpec = new JobQueueFilterSpecification(projectId, _inProgressJobStatus);
var inProgressJob = await _jobQueueRepository.GetSingleBySpec(inProgressJobSpec, cancellationToken);
if (inProgressJob != null)
{
if (project.Status == ProjectStatusFilterType.Deleting && jobType != JobType.Delete)
{
return(inProgressJob.Id);
}
throw new JobQueueInProgressException(projectId);
}
JobDefinition jobDefinition = null;
if (jobDefinitionId.HasValue)
{
jobDefinition = await _jobDefinitionService.GetJobDefinitionById(jobDefinitionId.Value, cancellationToken);
await ValidateJobTasks(jobDefinition, cancellationToken);
}
// detemine the jobQueue type if it is not provided by the caller
if (string.IsNullOrWhiteSpace(jobType))
{
jobType = JobType.Create;
var hasBeenCreatedBefore = (await _jobQueueRepository.CountBySpec(new JobQueueFilterSpecification(projectId, new string[] { JobStatus.Completed }))) > 0;
if (hasBeenCreatedBefore)
{
jobType = JobType.Update;
}
}
var newJobQueue = new JobQueue
{
ProjectId = projectId,
Code = await GetNextJobCode(),
OriginUrl = originUrl,
JobType = jobType,
JobDefinitionId = jobDefinition?.Id,
JobDefinitionName = jobDefinition?.Name,
IsDeletion = jobDefinition?.IsDeletion ?? false,
Status = JobStatus.Queued
};
return(await _jobQueueRepository.Create(newJobQueue, cancellationToken));
}
void SetupWorld(int width, int height)
{
jobQueue = new JobQueue();
Width = width;
Height = height;
tiles = new Tile[Width,Height];
for (int x = 0; x < Width; x++) {
for (int y = 0; y < Height; y++) {
tiles[x,y] = new Tile(this, x, y);
tiles[x,y].RegisterTileTypeChangedCallback( OnTileChanged );
}
}
Debug.Log ("World created with " + (Width*Height) + " tiles.");
CreateFurniturePrototypes();
characters = new List<Character>();
furnitures = new List<Furniture>();
}
/// <summary>
/// Listens to the commands from server
/// </summary>
/// <param name="testHostManagerFactory">the test host manager.</param>
public void ProcessRequests(ITestHostManagerFactory testHostManagerFactory)
{
bool isSessionEnd = false;
var jobQueue = new JobQueue <Action>(
action => { action(); },
"TestHostOperationQueue",
500,
25000000,
true,
(message) => EqtTrace.Error(message));
do
{
var message = this.communicationManager.ReceiveMessage();
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("TestRequestHandler.ProcessRequests: received message: {0}", message);
}
switch (message.MessageType)
{
case MessageType.VersionCheck:
var version = this.dataSerializer.DeserializePayload <int>(message);
this.protocolVersion = Math.Min(version, highestSupportedVersion);
this.communicationManager.SendMessage(MessageType.VersionCheck, this.protocolVersion);
// Can only do this after InitializeCommunication because TestHost cannot "Send Log" unless communications are initialized
if (!string.IsNullOrEmpty(EqtTrace.LogFile))
{
this.SendLog(TestMessageLevel.Informational, string.Format(CrossPlatResources.TesthostDiagLogOutputFile, EqtTrace.LogFile));
}
else if (!string.IsNullOrEmpty(EqtTrace.ErrorOnInitialization))
{
this.SendLog(TestMessageLevel.Warning, EqtTrace.ErrorOnInitialization);
}
break;
case MessageType.DiscoveryInitialize:
{
EqtTrace.Info("Discovery Session Initialize.");
var pathToAdditionalExtensions = this.dataSerializer.DeserializePayload <IEnumerable <string> >(message);
jobQueue.QueueJob(
() => testHostManagerFactory.GetDiscoveryManager()
.Initialize(pathToAdditionalExtensions),
0);
break;
}
case MessageType.StartDiscovery:
{
EqtTrace.Info("Discovery started.");
var discoveryEventsHandler = new TestDiscoveryEventHandler(this);
var discoveryCriteria = this.dataSerializer.DeserializePayload <DiscoveryCriteria>(message);
jobQueue.QueueJob(
() => testHostManagerFactory.GetDiscoveryManager()
.DiscoverTests(discoveryCriteria, discoveryEventsHandler),
0);
break;
}
case MessageType.ExecutionInitialize:
{
EqtTrace.Info("Discovery Session Initialize.");
var pathToAdditionalExtensions = this.dataSerializer.DeserializePayload <IEnumerable <string> >(message);
jobQueue.QueueJob(
() => testHostManagerFactory.GetExecutionManager()
.Initialize(pathToAdditionalExtensions),
0);
break;
}
case MessageType.StartTestExecutionWithSources:
{
EqtTrace.Info("Execution started.");
var testRunEventsHandler = new TestRunEventsHandler(this);
var testRunCriteriaWithSources = this.dataSerializer.DeserializePayload <TestRunCriteriaWithSources>(message);
jobQueue.QueueJob(
() =>
testHostManagerFactory.GetExecutionManager()
.StartTestRun(
testRunCriteriaWithSources.AdapterSourceMap,
testRunCriteriaWithSources.Package,
testRunCriteriaWithSources.RunSettings,
testRunCriteriaWithSources.TestExecutionContext,
this.GetTestCaseEventsHandler(testRunCriteriaWithSources.RunSettings),
testRunEventsHandler),
0);
break;
}
case MessageType.StartTestExecutionWithTests:
{
EqtTrace.Info("Execution started.");
var testRunEventsHandler = new TestRunEventsHandler(this);
var testRunCriteriaWithTests =
this.communicationManager.DeserializePayload <TestRunCriteriaWithTests>(message);
jobQueue.QueueJob(
() =>
testHostManagerFactory.GetExecutionManager()
.StartTestRun(
testRunCriteriaWithTests.Tests,
testRunCriteriaWithTests.Package,
testRunCriteriaWithTests.RunSettings,
testRunCriteriaWithTests.TestExecutionContext,
this.GetTestCaseEventsHandler(testRunCriteriaWithTests.RunSettings),
testRunEventsHandler),
0);
break;
}
case MessageType.CancelTestRun:
jobQueue.Pause();
testHostManagerFactory.GetExecutionManager().Cancel();
break;
case MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback:
this.onAckMessageRecieved?.Invoke(message);
break;
case MessageType.AbortTestRun:
jobQueue.Pause();
testHostManagerFactory.GetExecutionManager().Abort();
break;
case MessageType.SessionEnd:
{
EqtTrace.Info("Session End message received from server. Closing the connection.");
isSessionEnd = true;
this.Close();
break;
}
case MessageType.SessionAbort:
{
// Dont do anything for now.
break;
}
default:
{
EqtTrace.Info("Invalid Message types");
break;
}
}
}while (!isSessionEnd);
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
queues.ThrowIfNull("queues");
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", "queues");
}
JobQueue fetchedJob = null;
var fetchConditions = new Expression <Func <Hangfire_JobQueues.Mapping, bool> >[]
{
job => job.FetchedAt == null,
job => job.FetchedAt < DateTime.UtcNow.AddSeconds(_options.InvisibilityTimeout.Negate().TotalSeconds)
};
var currentQueryIndex = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
var fetchCondition = fetchConditions[currentQueryIndex];
foreach (var queue in queues)
{
using (var repository = _storage.Repository.OpenSession())
{
foreach (var job in repository.Query <Hangfire_JobQueues.Mapping, Hangfire_JobQueues>()
.Where(fetchCondition)
.Where(job => job.Queue == queue)
.OfType <JobQueue>())
{
job.FetchedAt = DateTime.UtcNow;
try
{
// Did someone else already picked it up?
repository.Advanced.UseOptimisticConcurrency = true;
repository.SaveChanges();
fetchedJob = job;
break;
}
catch (ConcurrencyException)
{
repository.Advanced.Evict(job); // Avoid subsequent concurrency exceptions
}
}
}
if (fetchedJob != null)
{
break;
}
}
if (fetchedJob == null)
{
if (currentQueryIndex == fetchConditions.Length - 1)
{
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
}
currentQueryIndex = (currentQueryIndex + 1) % fetchConditions.Length;
}while (fetchedJob == null);
return(new RavenFetchedJob(_storage, fetchedJob));
}
public PublicFolderSyncJob(JobQueue queue, OrganizationId orgId, Guid contentMailboxGuid, bool executeReconcileFolders) : base(queue, queue.Configuration, string.Empty)
{
this.OrganizationId = orgId;
this.ContentMailboxGuid = contentMailboxGuid;
this.ExecuteReconcileFolders = executeReconcileFolders;
}
/// <summary>
/// Constructor
/// </summary>
public ScrapeWorker()
{
_JobQueue = JobQueue.Instance;
_Cache = Cache.Instance;
}
public void TestDataSet2()
{
var input = new DataSource();
input.AppendLine("10 100");
var inputValues = "124860658 388437511 753484620 349021732 311346104 235543106 665655446 28787989 706718118 409836312 217716719 757274700 609723717 880970735 972393187 246159983 318988174 209495228 854708169 945600937 773832664 587887000 531713892 734781348 603087775 148283412 195634719 968633747 697254794 304163856 554172907 197744495 261204530 641309055 773073192 463418708 59676768 16042361 210106931 901997880 220470855 647104348 163515452 27308711 836338869 505101921 397086591 126041010 704685424 48832532 944295743 840261083 407178084 723373230 242749954 62738878 445028313 734727516 370425459 607137327 541789278 281002380 548695538 651178045 638430458 981678371 648753077 417312222 446493640 201544143 293197772 298610124 31821879 46071794 509690783 183827382 867731980 524516363 376504571 748818121 36366377 404131214 128632009 535716196 470711551 19833703 516847878 422344417 453049973 58419678 175133498 967886806 49897195 188342011 272087192 798530288 210486166 836411405 909200386 561566778";
input.AppendLine(inputValues);
var output = new DataSource();
output.AppendLine("0 0");
output.AppendLine("1 0");
output.AppendLine("2 0");
output.AppendLine("3 0");
output.AppendLine("4 0");
output.AppendLine("5 0");
output.AppendLine("6 0");
output.AppendLine("7 0");
output.AppendLine("8 0");
output.AppendLine("9 0");
output.AppendLine("7 28787989");
output.AppendLine("0 124860658");
output.AppendLine("5 235543106");
output.AppendLine("7 246504708");
output.AppendLine("4 311346104");
output.AppendLine("3 349021732");
output.AppendLine("1 388437511");
output.AppendLine("9 409836312");
output.AppendLine("3 595181715");
output.AppendLine("9 619331540");
output.AppendLine("6 665655446");
output.AppendLine("8 706718118");
output.AppendLine("1 707425685");
output.AppendLine("2 753484620");
output.AppendLine("5 845266823");
output.AppendLine("0 882135358");
output.AppendLine("0 1030418770");
output.AppendLine("7 1127475443");
output.AppendLine("0 1226053489");
output.AppendLine("1 1239139577");
output.AppendLine("4 1283739291");
output.AppendLine("8 1294605118");
output.AppendLine("6 1439488110");
output.AppendLine("5 1448354598");
output.AppendLine("3 1449889884");
output.AppendLine("2 1488265968");
output.AppendLine("8 1492349613");
output.AppendLine("1 1543303433");
output.AppendLine("8 1552026381");
output.AppendLine("1 1559345794");
output.AppendLine("9 1564932477");
output.AppendLine("6 1700692640");
output.AppendLine("8 1762133312");
output.AppendLine("9 1785403332");
output.AppendLine("9 1812712043");
output.AppendLine("4 1837912198");
output.AppendLine("0 1923308283");
output.AppendLine("8 1925648764");
output.AppendLine("2 1951684676");
output.AppendLine("8 2051689774");
output.AppendLine("5 2089663653");
output.AppendLine("7 2096109190");
output.AppendLine("8 2100522306");
output.AppendLine("3 2222963076");
output.AppendLine("0 2320394874");
output.AppendLine("4 2343014119");
output.AppendLine("6 2347796988");
output.AppendLine("4 2405752997");
output.AppendLine("1 2461343674");
output.AppendLine("8 2507700390");
output.AppendLine("0 2563144828");
output.AppendLine("9 2649050912");
output.AppendLine("2 2656370100");
output.AppendLine("6 2792825301");
output.AppendLine("1 2831769133");
output.AppendLine("9 2930053292");
output.AppendLine("7 2936370273");
output.AppendLine("3 2946336306");
output.AppendLine("5 3033959396");
output.AppendLine("0 3104934106");
output.AppendLine("8 3114837717");
output.AppendLine("4 3140480513");
output.AppendLine("2 3205065638");
output.AppendLine("2 3236887517");
output.AppendLine("2 3282959311");
output.AppendLine("0 3306478249");
output.AppendLine("3 3363648528");
output.AppendLine("8 3408035489");
output.AppendLine("4 3439090637");
output.AppendLine("6 3444003346");
output.AppendLine("1 3470199591");
output.AppendLine("5 3480453036");
output.AppendLine("0 3490305631");
output.AppendLine("1 3506565968");
output.AppendLine("7 3585123350");
output.AppendLine("0 3618937640");
output.AppendLine("0 3638771343");
output.AppendLine("2 3792650094");
output.AppendLine("4 3815595208");
output.AppendLine("5 3884584250");
output.AppendLine("9 3911731663");
output.AppendLine("8 3932551852");
output.AppendLine("5 3943003928");
output.AppendLine("5 3992901123");
output.AppendLine("1 4042282164");
output.AppendLine("7 4055834901");
output.AppendLine("9 4086865161");
output.AppendLine("0 4155619221");
output.AppendLine("5 4181243134");
output.AppendLine("6 4192821467");
var assingment = new JobQueue();
Assert.Equal(output.ToString(), assingment.Execute(input));
}
protected TestRequestHandler(TestHostConnectionInfo connectionInfo, ICommunicationEndPoint communicationEndpoint, IDataSerializer dataSerializer, JobQueue <Action> jobQueue, Action <Message> onAckMessageRecieved)
{
this.communicationEndPoint = communicationEndpoint;
this.connectionInfo = connectionInfo;
this.dataSerializer = dataSerializer;
this.requestSenderConnected = new ManualResetEventSlim(false);
this.testHostManagerFactoryReady = new ManualResetEventSlim(false);
this.sessionCompleted = new ManualResetEventSlim(false);
this.onAckMessageRecieved = onAckMessageRecieved;
this.jobQueue = jobQueue;
}
public DocumentSyncJob(JobQueue queue, Configuration config, TeamMailboxSyncInfo syncInfoEntry, string clientString, SyncOption syncOption) : base(queue, config, syncInfoEntry, clientString, syncOption)
{
this.loggingComponent = ProtocolLog.Component.DocumentSync;
}
public override void CancelJob()
{
Owner.Collectable = null;
JobQueue.Enqueue(new ButcherCollectJob(collectable, collectable.Transform.position));
}
public void GetFactoryFromDataQueueHandler(string dataQueueHandler)
{
var factory = JobQueue.GetFactoryFromDataQueueHandler(dataQueueHandler);
Assert.NotNull(factory);
}
private void Jobs_JobQueueAdded(JobQueue theQueue, string addedQueue)
{
Debug.Log("Job priority button created");
CreateJobButton(addedQueue);
}
// Not sure how else to do this, lots of duplicate code but I want to force the compiler to see at runtime what exact subtype is calling the method // For some reason the compiler is stupid and doesn't call Enqueue with the correct type if we do it from here (even if it's a subtype!) so we have to implement this method // in every single subclass which is a lot of duplicate code but oh well /// <summary> /// Enqueues the item from the subclass, this ensures that the program knows at runtime what the specific type of the job is and doesn't just enqueue the job as a "Job" /// </summary> /// <param name="theQueue">The queue to enqueue the job in</param> /// <param name="firstItem">Whether or not this is the first item getting enqueued after deserialisation</param> public abstract void EnqueueFromSubclass(JobQueue theQueue, bool firstItem = false);
public TestableTestRequestHandler(TestHostConnectionInfo testHostConnectionInfo, ICommunicationEndpointFactory communicationEndpointFactory, IDataSerializer dataSerializer, JobQueue <Action> jobQueue)
: base(testHostConnectionInfo, communicationEndpointFactory, dataSerializer, jobQueue, OnAckMessageReceived)
{
}
private void LaunchLean(string id)
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
if (!string.IsNullOrEmpty(_config.TransactionLog))
{
var filename = _config.TransactionLog;
filename = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Path.GetFileNameWithoutExtension(filename) + id + Path.GetExtension(filename));
Config.Set("transaction-log", filename);
}
Config.Set("api-handler", nameof(EmptyApiHandler));
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
//Composer.Instance.Reset();
//todo: instance logging
//var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + id + ".txt";
Log.LogHandler = LogSingleton.Instance;
var jobQueue = new JobQueue();
var manager = new LocalLeanManager();
var systemHandlers = new LeanEngineSystemHandlers(
jobQueue,
new EmptyApiHandler(),
new QuantConnect.Messaging.Messaging(),
manager);
systemHandlers.Initialize();
var map = new LocalDiskMapFileProvider();
var results = new OptimizerResultHandler();
var transactions = new BacktestingTransactionHandler();
var dataFeed = new FileSystemDataFeed();
var realTime = new BacktestingRealTimeHandler();
var data = new DefaultDataProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
results,
new ConsoleSetupHandler(),
dataFeed,
transactions,
realTime,
map,
new LocalDiskFactorFileProvider(map),
data,
new OptimizerAlphaHandler(),
new EmptyObjectStore());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//mark job with id. Is set on algorithm in OptimizerAlphaHandler
job.BacktestId = id;
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
Engine engine;
AlgorithmManager algorithmManager;
try
{
algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
using (var workerThread = new MultipleWorkerThread())
{
engine.Run(job, algorithmManager, algorithmPath, workerThread);
}
}
finally
{
// clean up resources
Composer.Instance.Reset();
results.Charts.Clear();
results.Messages.Clear();
results.Algorithm.Transactions.TransactionRecord.Clear();
var closedTrades = (List <Trade>) typeof(TradeBuilder).GetField("_closedTrades", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(results.Algorithm.TradeBuilder);
closedTrades.Clear();
results.Algorithm.HistoryProvider = null;
results.Algorithm = null;
transactions.Orders.Clear();
transactions.OrderTickets.Clear();
manager.Dispose();
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
results = null;
dataFeed = null;
transactions = null;
realTime = null;
data = null;
map = null;
systemHandlers = null;
leanEngineAlgorithmHandlers = null;
algorithmManager = null;
engine = null;
job = null;
jobQueue = null;
manager = null;
}
}
public override async Task Run(string[] args)
{
// Parse the arguments.
if (!ParseArguments(args, out uint blockSizeBytes, out uint totalBlocks, out uint numInstances, out string blobName, out string containerName))
{
// If invalid arguments were provided, exit the test.
return;
}
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
uint startingBlockId = 0;
uint numBlocksPerPass = totalBlocks / numInstances;
uint remainder = totalBlocks % numInstances;
Task[] tasks = new Task[numInstances];
HashSet <Guid> operationIDs = new HashSet <Guid>();
// Assign the Work
for (uint i = 0; i < numInstances; ++i)
{
uint numBlocks = numBlocksPerPass;
// Evenly distribute remaining blocks across instances such that no instance differs by more than 1 block.
if (remainder > 0)
{
numBlocks++;
remainder--;
}
Guid opId = Guid.NewGuid();
operationIDs.Add(opId);
PutBlobOperation operation = new PutBlobOperation(opId, blobName, containerName, startingBlockId, blockSizeBytes, numBlocks);
TestMsg putBlobMsg = new TestMsg(Environment.MachineName, operation);
tasks[i] = JobQueue.AddMessageAsync(CreateJobQueueMessage(putBlobMsg));
// Update the starting position.
startingBlockId += numBlocks;
}
await Task.WhenAll(tasks);
// Report timing status (NOTE: omits time taken to execute PutBlockList).
await ReportStatus(operationIDs, ((long)totalBlocks) *blockSizeBytes);
// Commit the blob.
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
try
{
await FinalizeBlob(container, blobName, totalBlocks, false);
Console.WriteLine("Successfully committed the block list.");
}
catch (Exception ex)
{
Console.WriteLine($"[ERROR] Could not commit the block list. Details: {ex.Message}");
}
}
public HomeController(BackgroundTaskQueue backgroundJobQueue, JobQueue <SampleQueueItem> queue)
{
_backgroundJobQueue = backgroundJobQueue;
_queue = queue;
}
internal static void Initialize()
{
jobs = new JobQueue();
}
// Start is called before the first frame update
new private void Start()
{
deliveredResources = new List <ICollectable>();
jobQueue = FindObjectOfType <JobQueue>();
}
public SJF(JobQueue workQueue) : base(workQueue, new JobByArrivalTimeComparer())
{
}
public RoundRobin(JobQueue workQueue, int quantum = 0) : base(workQueue, new JobByArrivalTimeComparer())
{
Quantum = quantum;
}
private LiteJob CreateJobInState(HangfireDbContext database, int jobId, string stateName, Func <LiteJob, LiteJob> visitor = null)
{
var job = Job.FromExpression(() => SampleMethod("wrong"));
Dictionary <string, string> stateData;
if (stateName == EnqueuedState.StateName)
{
stateData = new Dictionary <string, string> {
["EnqueuedAt"] = $"{DateTime.Now:o}"
};
}
else if (stateName == ProcessingState.StateName)
{
stateData = new Dictionary <string, string>
{
["ServerId"] = Guid.NewGuid().ToString(),
["StartedAt"] = JobHelper.SerializeDateTime(DateTime.Now.Subtract(TimeSpan.FromMilliseconds(500)))
};
}
else
{
stateData = new Dictionary <string, string>();
}
var jobState = new LiteState()
{
JobId = jobId,
Name = stateName,
Reason = null,
CreatedAt = DateTime.Now,
Data = stateData
};
var liteJob = new LiteJob
{
Id = jobId,
InvocationData = JobHelper.ToJson(InvocationData.Serialize(job)),
Arguments = "[\"\\\"Arguments\\\"\"]",
StateName = stateName,
CreatedAt = DateTime.Now,
StateHistory = new List <LiteState> {
jobState
}
};
if (visitor != null)
{
liteJob = visitor(liteJob);
}
database.Job.Insert(liteJob);
var jobQueueDto = new JobQueue
{
FetchedAt = null,
JobId = jobId,
Queue = DefaultQueue
};
if (stateName == FetchedStateName)
{
jobQueueDto.FetchedAt = DateTime.Now;
}
database.JobQueue.Insert(jobQueueDto);
return(liteJob);
}
public IEnumerator CreateImpl()
{
try
{
using (m_queue = new JobQueue(8))
{
Stopwatch sw = new Stopwatch();
AssetDatabase.Refresh();
AssetDatabase.SaveAssets();
sw.Reset();
sw.Start();
EditorUtility.DisplayProgressBar("Bake HLOD", "Initialize Bake", 0.0f);
TerrainData data = m_hlod.TerrainData;
m_size = data.size;
m_heightmap = new Heightmap(data.heightmapResolution, data.heightmapResolution, data.size,
data.GetHeights(0, 0, data.heightmapResolution, data.heightmapResolution));
string materialPath = AssetDatabase.GUIDToAssetPath(m_hlod.MaterialGUID);
m_terrainMaterial = AssetDatabase.LoadAssetAtPath <Material>(materialPath);
if (m_terrainMaterial == null)
{
m_terrainMaterial = new Material(Shader.Find("Lightweight Render Pipeline/Lit-Terrain-HLOD-High"));
}
m_terrainMaterialInstanceId = m_terrainMaterial.GetInstanceID();
m_terrainMaterialName = m_terrainMaterial.name;
materialPath = AssetDatabase.GUIDToAssetPath(m_hlod.MaterialLowGUID);
m_terrainMaterialLow = AssetDatabase.LoadAssetAtPath <Material>(materialPath);
if (m_terrainMaterialLow == null)
{
m_terrainMaterialLow = new Material(Shader.Find("Lightweight Render Pipeline/Lit-Terrain-HLOD-Low"));
}
m_terrainMaterialLowInstanceId = m_terrainMaterialLow.GetInstanceID();
m_terrainMaterialLowName = m_terrainMaterialLow.name;
using (m_alphamaps = new DisposableList <WorkingTexture>())
using (m_layers = new DisposableList <Layer>())
{
for (int i = 0; i < data.alphamapTextures.Length; ++i)
{
m_alphamaps.Add(new WorkingTexture(Allocator.Persistent, data.alphamapTextures[i]));
}
for (int i = 0; i < data.terrainLayers.Length; ++i)
{
m_layers.Add(new Layer(data.terrainLayers[i], m_hlod.ChunkSize));
}
QuadTreeSpaceSplitter splitter =
new QuadTreeSpaceSplitter(0.0f);
SpaceNode rootNode = splitter.CreateSpaceTree(m_hlod.GetBounds(), m_hlod.ChunkSize * 2.0f,
m_hlod.transform.position, null, progress => { });
EditorUtility.DisplayProgressBar("Bake HLOD", "Create mesh", 0.0f);
using (DisposableList <HLODBuildInfo> buildInfos = CreateBuildInfo(data, rootNode))
{
yield return(m_queue.WaitFinish());
//Write material & textures
for (int i = 0; i < buildInfos.Count; ++i)
{
int curIndex = i;
m_queue.EnqueueJob(() =>
{
ISimplifier simplifier = (ISimplifier)Activator.CreateInstance(m_hlod.SimplifierType,
new object[] { m_hlod.SimplifierOptions });
simplifier.SimplifyImmidiate(buildInfos[curIndex]);
});
}
EditorUtility.DisplayProgressBar("Bake HLOD", "Simplify meshes", 0.0f);
yield return(m_queue.WaitFinish());
Debug.Log("[TerrainHLOD] Simplify: " + sw.Elapsed.ToString("g"));
sw.Reset();
sw.Start();
EditorUtility.DisplayProgressBar("Bake HLOD", "Make border", 0.0f);
for (int i = 0; i < buildInfos.Count; ++i)
{
HLODBuildInfo info = buildInfos[i];
m_queue.EnqueueJob(() =>
{
for (int oi = 0; oi < info.WorkingObjects.Count; ++oi)
{
WorkingObject o = info.WorkingObjects[oi];
int borderVertexCount = m_hlod.BorderVertexCount * Mathf.RoundToInt(Mathf.Pow(2.0f, (float)info.Distances[oi]));
using (WorkingMesh m = MakeBorder(o.Mesh, info.Heightmap, borderVertexCount))
{
ReampUV(m, info.Heightmap);
o.SetMesh(MakeFillHoleMesh(m));
}
}
});
}
yield return(m_queue.WaitFinish());
Debug.Log("[TerrainHLOD] Make Border: " + sw.Elapsed.ToString("g"));
sw.Reset();
sw.Start();
for (int i = 0; i < buildInfos.Count; ++i)
{
SpaceNode node = buildInfos[i].Target;
HLODBuildInfo info = buildInfos[i];
if (node.HasChild() == false)
{
SpaceNode parent = node.ParentNode;
node.ParentNode = null;
GameObject go = new GameObject(buildInfos[i].Name);
for (int wi = 0; wi < info.WorkingObjects.Count; ++wi)
{
WorkingObject wo = info.WorkingObjects[wi];
GameObject targetGO = null;
if (wi == 0)
{
targetGO = go;
}
else
{
targetGO = new GameObject(wi.ToString());
targetGO.transform.SetParent(go.transform, false);
}
List <Material> materials = new List <Material>();
for (int mi = 0; mi < wo.Materials.Count; ++mi)
{
WorkingMaterial wm = wo.Materials[mi];
if (wm.NeedWrite() == false)
{
materials.Add(wm.ToMaterial());
continue;
}
Material mat = new Material(wm.ToMaterial());
string[] textureNames = wm.GetTextureNames();
for (int ti = 0; ti < textureNames.Length; ++ti)
{
WorkingTexture wt = wm.GetTexture(textureNames[ti]);
Texture2D tex = wt.ToTexture();
tex.wrapMode = wt.WrapMode;
mat.name = targetGO.name + "_Mat";
mat.SetTexture(textureNames[ti], tex);
}
mat.EnableKeyword("_NORMALMAP");
materials.Add(mat);
}
targetGO.AddComponent <MeshFilter>().sharedMesh = wo.Mesh.ToMesh();
var mr = targetGO.AddComponent <MeshRenderer>();
mr.sharedMaterials = materials.ToArray();
mr.lightProbeUsage = UnityEngine.Rendering.LightProbeUsage.Off;
}
go.transform.SetParent(m_hlod.transform, false);
m_hlod.AddGeneratedResource(go);
parent.Objects.Add(go);
buildInfos.RemoveAt(i);
i -= 1;
}
}
//controller
IStreamingBuilder builder =
(IStreamingBuilder)Activator.CreateInstance(m_hlod.StreamingType,
new object[] { m_hlod, m_hlod.StreamingOptions });
builder.Build(rootNode, buildInfos, m_hlod.gameObject, m_hlod.CullDistance, m_hlod.LODDistance, true, false,
progress =>
{
EditorUtility.DisplayProgressBar("Bake HLOD", "Storing results.",
0.75f + progress * 0.25f);
});
Debug.Log("[TerrainHLOD] Build: " + sw.Elapsed.ToString("g"));
}
}
EditorUtility.SetDirty(m_hlod.gameObject);
}
}
finally
{
EditorUtility.ClearProgressBar();
GC.Collect();
}
}
public JobQueueProvider(CouchbaseStorage storage)
{
queue = new JobQueue(storage);
monitoringQueue = new JobQueueMonitoringApi(storage);
}
public JobQueueFunctionalTests()
{
queue = new JobQueue();
testJob = new TestJob();
jobTestJob = null;
}
/// <summary>
/// Initializes the Api, Messaging, and JobQueue components
/// </summary>
public void Initialize()
{
Api.Initialize(Config.GetInt("job-user-id", 0), Config.Get("api-access-token", ""), Config.Get("data-folder"));
Notify.Initialize();
JobQueue.Initialize(Api);
}
public void RequestMesh(Region heightMap, MeshSettings meshSettings)
{
hasRequestedMesh = true;
JobQueue.Run(() => MeshGenerator.GenerateTerrainMesh(heightMap.values, meshSettings, lod), this.OnMeshDataReceived);
}
public LoadSolutionProjects()
{
jobs = new JobQueue(this);
}
private void LaunchLean(string id)
{
ConfigMerger.Merge(_config, id, this.GetType());
//todo: instance logging
//var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + id + ".txt";
Log.LogHandler = LogSingleton.Instance;
var jobQueue = new JobQueue();
var manager = new LocalLeanManager();
var systemHandlers = new LeanEngineSystemHandlers(
jobQueue,
new EmptyApiHandler(),
new QuantConnect.Messaging.Messaging(),
manager);
systemHandlers.Initialize();
var map = new LocalDiskMapFileProvider();
var results = new OptimizerResultHandler();
var transactions = new BacktestingTransactionHandler();
var dataFeed = new FileSystemDataFeed();
var realTime = new BacktestingRealTimeHandler();
var data = new DefaultDataProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
results,
new ConsoleSetupHandler(),
dataFeed,
transactions,
realTime,
map,
new LocalDiskFactorFileProvider(map),
data,
new OptimizerAlphaHandler(),
new EmptyObjectStore(),
new DataPermissionManager());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//mark job with id. Is set on algorithm in OptimizerAlphaHandler
job.BacktestId = id;
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
Engine engine;
AlgorithmManager algorithmManager;
try
{
algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
using (var workerThread = new MultipleWorkerThread())
{
engine.Run(job, algorithmManager, algorithmPath, workerThread);
}
}
finally
{
// clean up resources
Composer.Instance.Reset();
results.Charts.Clear();
results.Messages.Clear();
if (results.Algorithm != null)
{
results.Algorithm.Transactions.TransactionRecord.Clear();
results.Algorithm.SubscriptionManager.Subscriptions.SelectMany(s => s.Consolidators)?.ToList().ForEach(f =>
{
results.Algorithm.SubscriptionManager.RemoveConsolidator(f.WorkingData?.Symbol, f);
UnregisterAllEvents(f);
});
if (results.Algorithm is QCAlgorithm)
{
((QCAlgorithm)results.Algorithm).SubscriptionManager.Subscriptions.ToList().Clear();
}
if (_config.AlgorithmLanguage != "Python")
{
results.Algorithm.HistoryProvider = null;
}
var closedTrades = (List <Trade>) typeof(TradeBuilder).GetField("_closedTrades", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(results.Algorithm.TradeBuilder);
closedTrades.Clear();
results.Algorithm = null;
}
transactions.Orders.Clear();
transactions.OrderTickets.Clear();
manager.Dispose();
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
results = null;
dataFeed = null;
transactions = null;
realTime = null;
data = null;
map = null;
systemHandlers = null;
leanEngineAlgorithmHandlers = null;
algorithmManager = null;
engine = null;
job = null;
jobQueue = null;
manager = null;
}
}
public void Init()
{
_jobQueue = new JobQueue();
_jobQueue.Clear();
}
public jobQueueDTO toDto(JobQueue jobQueue)
{
jobQueueDTO jobQueueDto = new jobQueueDTO();
jobQueueDto.jobQueueCode = jobQueue.jobQueueCode;
jobQueueDto.businessType = jobQueue.businessType;
jobQueueDto.laneCode = jobQueue.laneCode;
jobQueueDto.laneName = jobQueue.laneName;
jobQueueDto.laneType = jobQueue.laneType;
jobQueueDto.direction = jobQueue.direction;
jobQueueDto.startTime = Convert.ToDateTime(jobQueue.startTime).ToString("s") + "Z";
jobQueueDto.endTime = Convert.ToDateTime(jobQueue.endTime).ToString("s") + "Z";
jobQueueDto.licensePlateNumber = jobQueue.licensePlateNumber;
jobQueueDto.ocrLicensePlateNumber = jobQueue.ocrLicensePlateNumber;
jobQueueDto.isOcrLicensePlateNumberSucceeded = jobQueue.isOcrLicensePlateNumberSucceeded;
jobQueueDto.rfidLicensePlateNumber = jobQueue.rfidLicensePlateNumber;
jobQueueDto.rfidCardNumber = jobQueue.rfidCardNumber;
jobQueueDto.icCardNumber = jobQueue.icCardNumber;
jobQueueDto.customsLockNumber = jobQueue.customsLockNumber;
jobQueueDto.totalWeight = jobQueue.totalWeight;
jobQueueDto.containerQuantity = jobQueue.containerQuantity;
jobQueueDto.releaseResult = jobQueue.releaseResult;
jobQueueDto.releaseRuleResults = jobQueue.releaseRuleResults;
foreach (var item in jobQueueDto.releaseRuleResults)
{
if (item.updateTime.EndsWith("Z") != true)
{
item.updateTime = DateTime.Now.ToString("s") + "Z";
}
}
jobQueueDto.releasedType = jobQueue.releasedType;
jobQueueDto.releasedBy = jobQueue.releasedBy;
jobQueueDto.releasedTime = Convert.ToDateTime(jobQueue.releasedTime).ToString("s") + "Z";
jobQueueDto.pictures = JsonConvert.SerializeObject(jobQueue.pictures);
jobQueueDto.processes = JsonConvert.SerializeObject(jobQueue.processes);
jobQueueDto.extras = JsonConvert.SerializeObject(jobQueue.extras);
jobQueueDto.createdBy = jobQueue.laneCode;
jobQueueDto.createdTime = Convert.ToDateTime(jobQueue.startTime).ToString("s") + "Z";
jobQueueDto.lastModifiedBy = jobQueue.laneCode;
jobQueueDto.lastModifiedTime = Convert.ToDateTime(jobQueue.endTime).ToString("s") + "Z";
List <ContainerDTO> list = new List <ContainerDTO>();
foreach (var item in jobQueue.containers)
{
ContainerDTO cdto = new ContainerDTO();
cdto.containerNumber = item.containerNumber;
cdto.createdBy = jobQueue.laneCode;
cdto.createdTime = DateTime.UtcNow.ToString("s") + "Z";
cdto.isOcrContainerNumberSucceeded = true;
cdto.jobQueueCode = jobQueue.jobQueueCode;
cdto.ocrContainerNumber = item.ocrContainerNumber;
cdto.lastModifiedBy = jobQueue.laneCode;
cdto.lastModifiedTime = DateTime.UtcNow.ToString("s") + "Z";
cdto.ocrIsoCode = item.isoCode;
cdto.damages = JsonConvert.SerializeObject(item.damages);
list.Add(cdto);
}
jobQueueDto.containers = list;
return(jobQueueDto);
}
internal static void Initialize()
{
jobs = new JobQueue();
}
public static void ConfigSettings()
{
JobConfiguration.AppSettings = Settings.Default;
JobExecutionServiceName = Settings.Default.JobExecutionServiceName;
JobQueue = JobExecutionService.JobQueueFinder.GetJobQueue(JobExecutionServiceName, null);
}
