private async Async.Task <(BucketConfig, WorkSet)?> BuildWorkSet(Task[] tasks) { var taskIds = tasks.Select(x => x.TaskId).ToHashSet(); var workUnits = new List <WorkUnit>(); BucketConfig?bucketConfig = null; foreach (var task in tasks) { if ((task.Config.PrereqTasks?.Count ?? 0) > 0) { // if all of the prereqs are in this bucket, they will be // scheduled together if (!taskIds.IsSupersetOf(task.Config.PrereqTasks !)) { if (!(await _taskOperations.CheckPrereqTasks(task))) { continue; } } } var result = await BuildWorkunit(task); if (result == null) { continue; } if (bucketConfig == null) { bucketConfig = result.Value.Item1; } else if (bucketConfig != result.Value.Item1) { throw new Exception($"bucket configs differ: {bucketConfig} VS {result.Value.Item1}"); } workUnits.Add(result.Value.Item2); } if (bucketConfig != null) { var setupUrl = await _containers.GetContainerSasUrl(bucketConfig.setupContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List) ?? throw new Exception("container not found"); var workSet = new WorkSet( Reboot: bucketConfig.reboot, Script: bucketConfig.setupScript != null, SetupUrl: setupUrl, WorkUnits: workUnits ); return(bucketConfig, workSet); } return(null); }
public async Task <bool> ScheduleWorkset(Pool pool, WorkSet workSet) { if (pool.State == PoolState.Shutdown || pool.State == PoolState.Halt) { return(false); } return(await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus)); }
static void Main(string[] args) { var items = new string[] { "1", "2", "3", "300" }; using (var outfile = File.AppendText("file.txt")) { using (var ws = new WorkSet <string>(x => { lock (outfile) outfile.WriteLine(x); })) foreach (var item in items) { ws.Process(item); } } }
private async Async.Task <bool> ScheduleWorkset(WorkSet workSet, Pool pool, long count) { if (!PoolStateHelper.Available.Contains(pool.State)) { _logTracer.Info($"pool not available for work: {pool.Name} state: {pool.State}"); return(false); } for (var i = 0L; i < count; i++) { if (!await _poolOperations.ScheduleWorkset(pool, workSet)) { _logTracer.Error($"unable to schedule workset. pool:{pool.Name} workset: {workSet}"); return(false); } } return(true); }
private static void mainUpdateLoop(Object workSet) { WorkSet set = (WorkSet)workSet; List <Thing> list2 = set.list2; FieldInfo tickt = set.tickt; TickList __instance = set.__instance; Parallel.For(0, list2.Count, new ParallelOptions { MaxDegreeOfParallelism = ThreadingMod.NUM_THREADS_PER_MAP }, m => { if (!list2[m].Destroyed) { try { switch (((TickerType)tickt.GetValue(__instance))) { case TickerType.Normal: list2[m].Tick(); break; case TickerType.Rare: list2[m].TickRare(); break; case TickerType.Long: list2[m].TickLong(); break; } } catch (Exception ex) { if (Prefs.DevMode) { Log.Error("Exception ticking " + list2[m].ToString() + ": " + ex.ToString()); } } } }); loopDone = true; loopPauser.Set(); }
public void Run(IEnumerable <GallioAction> actions) { // NOTE: This method has been optimized to minimize the total stack depth of the action // by inlining blocks on the critical path that had previously been factored out. if (actions == null) { throw new ArgumentNullException("actions"); } // Copy the queue of actions to process from the enumeration. var actionQueue = new Queue <GallioAction>(); foreach (GallioAction action in actions) { if (action == null) { throw new ArgumentNullException("actions"); } actionQueue.Enqueue(action); } // Short-circuit when no actions to run to satisfy our invariant for pending work sets. if (actionQueue.Count == 0) { return; } // Add this work set to the list of pending work sets. WorkSet workSet; lock (syncRoot) { workSet = new WorkSet(actionQueue); pendingWorkSets.AddLast(workSet); activeThreads += 1; } // Loop until all actions in this work set are finished. for (; ;) { GallioAction nextAction; lock (syncRoot) { // Synchronize and exit if this work set is finished. if (!workSet.SyncHasPendingActions()) { activeThreads -= 1; while (workSet.SyncHasActionsInProgress()) { Monitor.Wait(syncRoot); } return; } // Prepare next action from this work set only. nextAction = workSet.SyncPrepareNextAction(); // Remove the work set from the list of pending work sets if it has no other pending actions. if (!workSet.SyncHasPendingActions()) { pendingWorkSets.Remove(workSet); } // Spawn more threads if needed for pending work sets. SyncSpawnBackgroundActionLoopIfNeeded(); } // Run the next action. try { nextAction(); } catch (Exception ex) { ReportUnhandledException(ex); } finally { lock (syncRoot) { workSet.SyncActionFinished(); Monitor.PulseAll(syncRoot); } } } }
public void Run(IEnumerable<Action> actions) { // NOTE: This method has been optimized to minimize the total stack depth of the action // by inlining blocks on the critical path that had previously been factored out. if (actions == null) throw new ArgumentNullException("actions"); // Copy the queue of actions to process from the enumeration. var actionQueue = new Queue<Action>(); foreach (Action action in actions) { if (action == null) throw new ArgumentNullException("actions"); actionQueue.Enqueue(action); } // Short-circuit when no actions to run to satisfy our invariant for pending work sets. if (actionQueue.Count == 0) return; // Add this work set to the list of pending work sets. WorkSet workSet; lock (syncRoot) { workSet = new WorkSet(actionQueue); pendingWorkSets.AddLast(workSet); activeThreads += 1; } // Loop until all actions in this work set are finished. for (; ; ) { Action nextAction; lock (syncRoot) { // Synchronize and exit if this work set is finished. if (!workSet.SyncHasPendingActions()) { activeThreads -= 1; while (workSet.SyncHasActionsInProgress()) Monitor.Wait(syncRoot); return; } // Prepare next action from this work set only. nextAction = workSet.SyncPrepareNextAction(); // Remove the work set from the list of pending work sets if it has no other pending actions. if (!workSet.SyncHasPendingActions()) pendingWorkSets.Remove(workSet); // Spawn more threads if needed for pending work sets. SyncSpawnBackgroundActionLoopIfNeeded(); } // Run the next action. try { nextAction(); } catch (Exception ex) { ReportUnhandledException(ex); } finally { lock (syncRoot) { workSet.SyncActionFinished(); Monitor.PulseAll(syncRoot); } } } }
public static TemplateList ReadFile(string templateListFilePath, FilterProcessor contentsFilter) { if (string.IsNullOrWhiteSpace(templateListFilePath)) { return(null); } TemplateList listData = new TemplateList(); try { using (FileStream fs = new FileStream(templateListFilePath, FileMode.Open, FileAccess.Read)) using (TextReader rd = new StreamReader(fs)) { WorkSet currentTargetSet = listData.mainWorkSet; string lineBuffer = rd.ReadLine(); while (lineBuffer != null) { if (string.IsNullOrWhiteSpace(lineBuffer) || lineBuffer.StartsWith('#')) { lineBuffer = rd.ReadLine(); continue; } lineBuffer = contentsFilter.Filter(lineBuffer).Trim(); string[] lineTokens = lineBuffer.Split(':', StringSplitOptions.TrimEntries); if (lineTokens.Length > 0) { if (lineTokens[0].StartsWith('[')) { // Section Descriptor string sectionName = _PopSectionName(lineTokens[0]); if (sectionName != null) { sectionName = sectionName.Trim(); if (sectionName == string.Empty) { currentTargetSet = listData.mainWorkSet; } else { if (!listData.conditionalWorkSets.ContainsKey(sectionName)) { listData.conditionalWorkSets.Add(sectionName, new WorkSet()); } currentTargetSet = listData.conditionalWorkSets[sectionName]; } } } else { switch (lineTokens[0]) { case "C": // Copy { if (lineTokens.Length > 1) { PathData srcPathData = PathData.ParsePathString(lineTokens[1]); PathData targetPathData = null; if (lineTokens.Length > 2) { targetPathData = PathData.ParsePathString(lineTokens[2]); } else { targetPathData = srcPathData; } if (lineTokens[1].EndsWith("/*")) { currentTargetSet.copyAllEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData)); } else { currentTargetSet.copyEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData)); } } } break; case "F": // Filter { if (lineTokens.Length > 1) { PathData srcPathData = PathData.ParsePathString(lineTokens[1]); PathData targetPathData = null; if (lineTokens.Length > 2) { targetPathData = PathData.ParsePathString(lineTokens[2]); } else { targetPathData = srcPathData; } currentTargetSet.filterEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData)); } } break; } } } lineBuffer = rd.ReadLine(); } } } catch { return(null); } return(listData); }