public static TreeScore Inference( int num_bootstraps, InputSample[][] resamples, Dictionary<AST.Range, InputSample> initial_inputs, Dictionary<AST.Address, string> initial_outputs, AST.Range[] input_arr, AST.Address[] output_arr, DAG dag, bool weighted, double significance, ProgBar pb) { // synchronization token object lock_token = new Object(); // init thread event notification array var mres = new ManualResetEvent[input_arr.Length]; // init job storage var ddjs = new DataDebugJob[input_arr.Length]; // init started jobs count var sjobs = 0; // init completed jobs count var cjobs = 0; // last-ditch effort flag bool last_try = false; // init score storage var scores = new TreeScore(); for (int i = 0; i < input_arr.Length; i++) { try { #region BOOTSTRAP // bootstrapping is done in the parent STA thread because // the .NET threading model prohibits thread pools (which // are MTA) from accessing STA COM objects directly. // alloc bootstrap storage for each output (f), for each resample (b) FunctionOutput<string>[][] bs = new FunctionOutput<string>[initial_outputs.Count][]; for (int f = 0; f < initial_outputs.Count; f++) { bs[f] = new FunctionOutput<string>[num_bootstraps]; } // init memoization table for input vector i var memo = new BootMemo(); // fetch the input range TreeNode var input = input_arr[i]; // fetch the input range COM object var com = dag.getCOMRefForRange(input).Range; // compute outputs // replace the values of the COM object with the jth bootstrap, // save all function outputs, and // restore the original input for (var b = 0; b < num_bootstraps; b++) { // lookup outputs from memo table; otherwise do replacement, compute outputs, store them in table, and return them FunctionOutput<string>[] fos = memo.FastReplace(com, dag, initial_inputs[input], resamples[i][b], output_arr, false); for (var f = 0; f < output_arr.Length; f++) { bs[f][b] = fos[f]; } } // restore the original inputs; faster to do once, after bootstrapping is done BootMemo.ReplaceExcelRange(com, initial_inputs[input]); // TODO: restore formulas if it turns out that they were overwrittern // this should never be the case #endregion BOOTSTRAP #region HYPOTHESIS_TEST // cancellation token mres[i] = new ManualResetEvent(false); // set up job ddjs[i] = new DataDebugJob( dag, bs, initial_outputs, input_arr[i], output_arr, weighted, significance, mres[i] ); sjobs++; // hand job to thread pool ThreadPool.QueueUserWorkItem(ddjs[i].threadPoolCallback, i); #endregion HYPOTHESIS_TEST // update progress bar pb.IncrementProgress(); } catch (System.OutOfMemoryException e) { if (!last_try) { // If there are no more jobs running, but // we still can't allocate memory, try invoking // GC and then trying again cjobs = mres.Count(mre => mre.WaitOne(0)); if (sjobs - cjobs == 0) { GC.Collect(); last_try = true; } } else { // we just don't have enough memory throw e; } // wait for any of the 0..i-1 work items // to complete and try again WaitHandle.WaitAny(mres.Take(i).ToArray()); } } // Do not proceed until all hypothesis tests are done. // WaitHandle.WaitAll cannot be called on an STA thread which // is why we call WaitOne in a loop. // Merge scores as data becomes available. for (int i = 0; i < input_arr.Length; i++) { mres[i].WaitOne(); scores = DictAdd(scores, ddjs[i].Result); } return scores; }
public static TreeScore Inference( int num_bootstraps, InputSample[][] resamples, Dictionary <AST.Range, InputSample> initial_inputs, Dictionary <AST.Address, string> initial_outputs, AST.Range[] input_arr, AST.Address[] output_arr, DAG dag, bool weighted, double significance, ProgBar pb) { // synchronization token object lock_token = new Object(); // init thread event notification array var mres = new ManualResetEvent[input_arr.Length]; // init job storage var ddjs = new DataDebugJob[input_arr.Length]; // init started jobs count var sjobs = 0; // init completed jobs count var cjobs = 0; // last-ditch effort flag bool last_try = false; // init score storage var scores = new TreeScore(); for (int i = 0; i < input_arr.Length; i++) { try { #region BOOTSTRAP // bootstrapping is done in the parent STA thread because // the .NET threading model prohibits thread pools (which // are MTA) from accessing STA COM objects directly. // alloc bootstrap storage for each output (f), for each resample (b) FunctionOutput <string>[][] bs = new FunctionOutput <string> [initial_outputs.Count][]; for (int f = 0; f < initial_outputs.Count; f++) { bs[f] = new FunctionOutput <string> [num_bootstraps]; } // init memoization table for input vector i var memo = new BootMemo(); // fetch the input range TreeNode var input = input_arr[i]; // fetch the input range COM object var com = dag.getCOMRefForRange(input).Range; // compute outputs // replace the values of the COM object with the jth bootstrap, // save all function outputs, and // restore the original input for (var b = 0; b < num_bootstraps; b++) { // lookup outputs from memo table; otherwise do replacement, compute outputs, store them in table, and return them FunctionOutput <string>[] fos = memo.FastReplace(com, dag, initial_inputs[input], resamples[i][b], output_arr, false); for (var f = 0; f < output_arr.Length; f++) { bs[f][b] = fos[f]; } } // restore the original inputs; faster to do once, after bootstrapping is done BootMemo.ReplaceExcelRange(com, initial_inputs[input]); // TODO: restore formulas if it turns out that they were overwrittern // this should never be the case #endregion BOOTSTRAP #region HYPOTHESIS_TEST // cancellation token mres[i] = new ManualResetEvent(false); // set up job ddjs[i] = new DataDebugJob( dag, bs, initial_outputs, input_arr[i], output_arr, weighted, significance, mres[i] ); sjobs++; // hand job to thread pool ThreadPool.QueueUserWorkItem(ddjs[i].threadPoolCallback, i); #endregion HYPOTHESIS_TEST // update progress bar pb.IncrementProgress(); } catch (System.OutOfMemoryException e) { if (!last_try) { // If there are no more jobs running, but // we still can't allocate memory, try invoking // GC and then trying again cjobs = mres.Count(mre => mre.WaitOne(0)); if (sjobs - cjobs == 0) { GC.Collect(); last_try = true; } } else { // we just don't have enough memory throw e; } // wait for any of the 0..i-1 work items // to complete and try again WaitHandle.WaitAny(mres.Take(i).ToArray()); } } // Do not proceed until all hypothesis tests are done. // WaitHandle.WaitAll cannot be called on an STA thread which // is why we call WaitOne in a loop. // Merge scores as data becomes available. for (int i = 0; i < input_arr.Length; i++) { mres[i].WaitOne(); scores = DictAdd(scores, ddjs[i].Result); } return(scores); }