public Mcsrch(int thread_num) { infoc = 0; stage1 = false; brackt = false; finit = 0.0; dginit = 0.0; dgtest = 0.0; width = 0.0; width1 = 0.0; stx = 0.0; fx = 0.0; dgx = 0.0; sty = 0.0; fy = 0.0; dgy = 0.0; stmin = 0.0; stmax = 0.0; #if NO_SUPPORT_PARALLEL_LIB #else parallelOption = new ParallelOptions(); parallelOption.MaxDegreeOfParallelism = thread_num; #endif }
/// <summary> /// Запускает на выполнение задачу по скачиванию песен и возвращает результат /// </summary> /// <param name="CancToken">Токен отмены операции</param> /// <param name="MaxDegreeOfParallelism">Максимальное количество потоков, которое будет использоваться для запросов к серверу. /// Если меньше 1, ограничение на количество потоков будет снято.</param> /// <returns></returns> public IDictionary<OneSongHeader, Exception> Start(CancellationToken CancToken, Int32 MaxDegreeOfParallelism) { if (MaxDegreeOfParallelism < 1) { MaxDegreeOfParallelism = -1; } ConcurrentDictionary<OneSongHeader, Exception> intermediate = new ConcurrentDictionary<OneSongHeader, Exception>(MaxDegreeOfParallelism, this._songs.Length); ParallelOptions opt = new ParallelOptions() { CancellationToken = CancToken, MaxDegreeOfParallelism = MaxDegreeOfParallelism }; try { ParallelLoopResult p_res = Parallel.ForEach(this._songs, opt, (OneSongHeader song, ParallelLoopState pls, Int64 i) => { if (pls.ShouldExitCurrentIteration) { pls.Stop(); } KeyValuePair<OneSongHeader, Exception> res = Core.DownloadAndSaveOneSong (song, this._userAgent, this._generateNewFilenames, this._filenameTemplate, this._folderPath, (Int32)i + 1); this.OnNext.Invoke(res.Key, res.Value); intermediate.TryAdd(res.Key, res.Value); } ); } catch (OperationCanceledException) { this.OnCancellation(intermediate.Count, this._songs.Length); CancToken.ThrowIfCancellationRequested(); } this.OnComplete.Invoke(intermediate); return intermediate; }
public object Solve() { int answer = 0; CancellationTokenSource tokenSource = new CancellationTokenSource(); var options = new ParallelOptions() { CancellationToken = tokenSource.Token, MaxDegreeOfParallelism = 4 }; try { Parallel.ForEach(TriangleNumbers().Skip(500), options, num => { var count = Divisors(num).Count(); if (count > 500) { tokenSource.Cancel(); answer = num; } }); } catch (OperationCanceledException) { // no op } return answer; }
public virtual async Task<CodeAction> GetFixAsync( ImmutableDictionary<Document, ImmutableArray<Diagnostic>> documentsAndDiagnosticsToFixMap, FixAllContext fixAllContext) { if (documentsAndDiagnosticsToFixMap != null && documentsAndDiagnosticsToFixMap.Any()) { fixAllContext.CancellationToken.ThrowIfCancellationRequested(); var documents = documentsAndDiagnosticsToFixMap.Keys.ToImmutableArray(); var fixesBag = new List<CodeAction>[documents.Length]; var options = new ParallelOptions() { CancellationToken = fixAllContext.CancellationToken }; Parallel.ForEach(documents, options, (document, state, index) => { fixAllContext.CancellationToken.ThrowIfCancellationRequested(); fixesBag[index] = new List<CodeAction>(); this.AddDocumentFixesAsync(document, documentsAndDiagnosticsToFixMap[document], fixesBag[index].Add, fixAllContext).Wait(fixAllContext.CancellationToken); }); if (fixesBag.Any(fixes => fixes.Count > 0)) { return await this.TryGetMergedFixAsync(fixesBag.SelectMany(i => i), fixAllContext).ConfigureAwait(false); } } return null; }
static void Main(string[] args) { int start = Environment.TickCount; int maxParallelUploads = 50; string urlForUploads = "http://localhost:8080"; string dirToUpload = Path.Combine(".", "default-to-upload"); if (args.Length > 0) { dirToUpload = args[0]; } ParallelOptions opts = new ParallelOptions(); opts.MaxDegreeOfParallelism = maxParallelUploads; Parallel.ForEach(Directory.EnumerateFiles(dirToUpload, "*", SearchOption.AllDirectories), opts, (filepath) => { Console.WriteLine("Uploading {0} on thread {1}...", filepath, Thread.CurrentThread.ManagedThreadId); WebClient webClient = new WebClient(); int sleepPeriodMs = 1000; bool retry = true; bool success = false; while (retry) { retry = false; try { webClient.UploadFile(urlForUploads, filepath); success = true; } catch (WebException e) { var r = (HttpWebResponse)e.Response; if (r != null && r.StatusCode == HttpStatusCode.ServiceUnavailable) { // We are overloading the server. Wait some time and try again. Console.WriteLine("Server is overloaded. Retrying in {0}ms...", sleepPeriodMs); Thread.Sleep(sleepPeriodMs); sleepPeriodMs *= 2; retry = true; } else { Console.WriteLine("Failed to upload file {0}. Error was: \n{1}.\nMoving on to next file.", filepath, e.ToString()); } } catch { Console.WriteLine("Unexpected error! Failed to upload file {0}. Moving on to next file.", filepath); } } if (success) { // The file was successfully uploaded to the server - delete it! File.Delete(filepath); } }); Console.WriteLine("Took {0} ticks to upload files.", Environment.TickCount - start); }
public Form1() { InitializeComponent(); cancellationTokenSource = new CancellationTokenSource(); options = new ParallelOptions(); options.CancellationToken = cancellationTokenSource.Token; }
private void Execute(IStatSource state, ParallelOptions parallelOptions) { state.Calculate(parallelOptions.CancellationToken); lock (_statLock) { _pendingStatJobs--; } }
private void DoWork(object unused) { try { var fileList = GetFiles(); var parallelOptions = new ParallelOptions(); parallelOptions.CancellationToken = _cancellationToken; parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount; Parallel.For(0, fileList.Count, parallelOptions, i => { string file = fileList[i]; IEnumerable<ParseItem> items = GetItems(file, _searchValue); foreach (ParseItem sel in items) { _cancellationToken.ThrowIfCancellationRequested(); _navigateToCallback.AddItem(new NavigateToItem(_searchValue, NavigateToItemKind.Field, "CSS", _searchValue, new CssGoToLineTag(sel, file), MatchKind.Exact, _providerFactory)); } var backgroundProgress = Interlocked.Increment(ref _backgroundProgress); _navigateToCallback.ReportProgress(backgroundProgress, fileList.Count); }); } catch { // Don't let exceptions from the background thread reach the ThreadPool. Swallow them // here and complete the navigate operation } finally { _navigateToCallback.Done(); } }
private static void CancellationOfParallelFor() { var nums = Enumerable.Range(0, 100000); var cts = new CancellationTokenSource(); var options = new ParallelOptions() { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; Console.WriteLine("Press any key to start, Press 'c' to cancel"); Console.ReadKey(); Task.Factory.StartNew(() => { if (Console.ReadKey().KeyChar == 'c') cts.Cancel(); }); try { Parallel.ForEach(nums, options, num => { var d = Math.Sqrt(num); Console.WriteLine("{0} on {1}", d, Thread.CurrentThread.ManagedThreadId); options.CancellationToken.ThrowIfCancellationRequested(); }); } catch (OperationCanceledException ex) { Console.WriteLine(ex.Message); } }
public void DroppingConnectionsTest() { var parallelOptions = new ParallelOptions { TaskScheduler = new ThreadPerTaskScheduler(), MaxDegreeOfParallelism = 1000 }; var clusterInfo = TestUtils.CcmSetup(1); var session = clusterInfo.Session; //For a node to be back up could take up to 60 seconds const int bringUpNodeMilliseconds = 60000; try { Action dropConnections = () => { session.Execute("SELECT * FROM system.schema_keyspaces"); TestUtils.CcmStopForce(clusterInfo, 1); Thread.Sleep(2000); TestUtils.CcmStart(clusterInfo, 1); }; Action query = () => { Thread.Sleep(bringUpNodeMilliseconds); //All the nodes should be up but the socket connections are not valid session.Execute("SELECT * FROM system.schema_keyspaces"); }; Parallel.Invoke(parallelOptions, dropConnections, query); } finally { TestUtils.CcmRemove(clusterInfo); } }
static void Foo1() { var list = new List<int>() {10,20,30,40,50 }; var options = new ParallelOptions(); var total = 0; var result = Parallel.For(0, list.Count, () => { Console.WriteLine("Thread"); return 1; }, (i, loop, j) => { Console.WriteLine("body"); Console.WriteLine("i= " + list[i] + " j=" + j); return list[i]; }, (i) => { Console.WriteLine("Foot"); Interlocked.Add(ref total,i); Console.WriteLine("total="+total); }); Console.WriteLine(result.IsCompleted); }
public virtual async Task<CodeAction> GetFixAsync( ImmutableDictionary<Document, ImmutableArray<Diagnostic>> documentsAndDiagnosticsToFixMap, FixAllContext fixAllContext) { if (documentsAndDiagnosticsToFixMap != null && documentsAndDiagnosticsToFixMap.Any()) { FixAllLogger.LogDiagnosticsStats(documentsAndDiagnosticsToFixMap); var fixesBag = new ConcurrentBag<CodeAction>(); using (Logger.LogBlock(FunctionId.CodeFixes_FixAllOccurrencesComputation_Fixes, fixAllContext.CancellationToken)) { fixAllContext.CancellationToken.ThrowIfCancellationRequested(); var documents = documentsAndDiagnosticsToFixMap.Keys.ToImmutableArray(); var options = new ParallelOptions() { CancellationToken = fixAllContext.CancellationToken }; Parallel.ForEach(documents, options, document => { fixAllContext.CancellationToken.ThrowIfCancellationRequested(); AddDocumentFixesAsync(document, documentsAndDiagnosticsToFixMap[document], fixesBag.Add, fixAllContext).Wait(fixAllContext.CancellationToken); }); } if (fixesBag.Any()) { using (Logger.LogBlock(FunctionId.CodeFixes_FixAllOccurrencesComputation_Merge, fixAllContext.CancellationToken)) { FixAllLogger.LogFixesToMergeStats(fixesBag); return await TryGetMergedFixAsync(fixesBag, fixAllContext).ConfigureAwait(false); } } } return null; }
public void SearchBattle() { var opt = new ParallelOptions(); opt.MaxDegreeOfParallelism = 4; Parallel.ForEach(Players.Where(p => !p.isBattle).OrderBy(p=>GB.Random.NextDouble()).ToList(), opt, () => new List<Battle>(), (player, pls, battles) => { if (Battles.Find(p => p != null && p.CanJoin(player.Udemae) && p.Join(player)) == null) { if (battles.Find(p => p != null && p.CanJoin(player.Udemae) && p.Join(player)) == null) { var battle = new Battle(player.Udemae); lock (this) { battles.Add(battle); battle.Join(player); } } } return battles; } , (battles) => { lock (this) { Battles.AddRange(battles); } } ); }
protected override void Run(CancellationToken cancellationToken) { this.cancellationToken = cancellationToken; parallelOptions = new ParallelOptions(); parallelOptions.MaxDegreeOfParallelism = DegreeOfParallelism; parallelOptions.CancellationToken = cancellationToken; Run(ExecutionStack); }
public void ExternalBreak(CancellationTokenSource cancellationTokenSource) { var cancellationToken = cancellationTokenSource.Token; var options = new ParallelOptions { CancellationToken = cancellationToken }; try { Parallel.For( 0, 100, options, i => { if (cancellationToken.IsCancellationRequested) { return; } Console.WriteLine(i); Thread.Sleep(500); }); } catch (OperationCanceledException ex) { Console.WriteLine(ex.Message); } }
public static void Main () { object count_lock = new object (); int count = 0; ParallelOptions options = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount * 4, }; Thread t1 = new Thread (() => { Parallel.ForEach (UntilTimeout (15 * 1000), options, _ => { using (Process p = Process.Start ("cat", "/dev/null")) { p.WaitForExit (); } lock (count_lock) { count += 1; if (count % (10) == 0) Console.Write ("."); if (count % (10 * 50) == 0) Console.WriteLine (); } }); }); t1.Start (); while (!t1.Join (0)) { try { using (Process p = Process.GetProcessById (1)); } catch (ArgumentException) { } } }
static void Main() { var tokenSource = new CancellationTokenSource(); var options = new ParallelOptions { CancellationToken = tokenSource.Token }; ThreadPool.QueueUserWorkItem(obj => { try { Parallel.For(0, 1000, options, i => DoWork(i, options.CancellationToken)); } catch (OperationCanceledException) { Console.WriteLine("Loop operation was cancelled"); } }); Thread.Sleep(250); Console.WriteLine("Cancelling work..."); tokenSource.Cancel(); Thread.Sleep(1000); }
async Task<IDictionary<string, CatalogItemSummary>> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken) { ConcurrentDictionary<string, CatalogItemSummary> pageItems = new ConcurrentDictionary<string, CatalogItemSummary>(); ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = _threads; var items = _batch.ToArray(); ConcurrentDictionary<CatalogItem, Uri> tmpPages = new ConcurrentDictionary<CatalogItem, Uri>(); Parallel.ForEach(items, options, item => { Uri resourceUri = null; try { item.TimeStamp = commitTimeStamp; item.CommitId = commitId; item.BaseAddress = Storage.BaseAddress; Uri catalogPageUri = CreateCatalogPage(item); //CommitItemComplete(catalogPageUri); resourceUri = item.GetItemAddress(); if (!tmpPages.TryAdd(item, catalogPageUri)) { throw new Exception("duplicate item"); } //if (catalogPageUri != null) //{ // Uri indexPageUri = CreateIndexEntry(item, catalogPageUri, commitId, commitTimeStamp); // CommitItemComplete(catalogPageUri, indexPageUri); //} //else //{ // Debug.Fail("Missing catalog content"); //} } catch (Exception e) { throw new Exception(string.Format("item uri: {0}", resourceUri == null ? "none" : resourceUri.AbsoluteUri), e); } }); // make sure to commit these in the correct order foreach (var item in items) { Uri pageUri = null; tmpPages.TryGetValue(item, out pageUri); CommitItemComplete(pageUri); } return pageItems; }
public static Info search(String path, String extension, string predicate) { CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); ParallelOptions options = new ParallelOptions(); options.CancellationToken = cancellationTokenSource.Token; Info info = new Info(); string[] files = Directory.GetFiles(path, extension, SearchOption.AllDirectories); info.totalextensions = files.Length; ParallelLoopResult loopResult = Parallel.For( 0, files.Length, options, (i, loopState) => { if (File.ReadAllText(files[i]).Contains(predicate)) { info.names.Add(files[i]); } } ); if (loopResult.IsCompleted) { info.total = info.names.Count; } return info; }
public static ParallelLoopResult For( int start, int end, ParallelOptions options, Action<int> body ) { // Following the HAPPY PATH :) // Problem : If the division result is a decimal number var segment = ( ( end - start ) / options.MaxDegreeOfParallelism ); var totalSplits = ( end - start ) / segment; Task task = new Task( () => { for ( int i = 0 ; i < totalSplits ; ++i ) { int i2 = i; Task.Factory.StartNew( () => { int i1 = i2; Console.WriteLine( "Start -> {0}", start + ( i1 * segment ) ); Console.WriteLine( "End -> {0}", start + segment + ( i1 * segment ) ); for ( int j = start + ( i1 * segment ) ; j < start + segment + ( i1 * segment ) ; ++j ) { body( j ); } }, TaskCreationOptions.AttachedToParent ); } } ); task.Start( options.TaskScheduler ); task.Wait( options.CancellationToken ); return Parallel.For(0, 0, (x) => Console.WriteLine( "Completed" ) ); }
static void DynamicSchedulingAndWorkStealing() { const int iterations = 10000; var options = new ParallelOptions();// { MaxDegreeOfParallelism = -1 }; var threadsUsed = new ConcurrentDictionary<int, int>(); var sw = new Stopwatch(); sw.Start(); Parallel.ForEach(Enumerable.Range(1, iterations), options, () => 0L, (val, pls, local) => { threadsUsed.TryAdd(Thread.CurrentThread.ManagedThreadId, 0); if (val < (iterations / 2)) { //for (int i = 0; i < 1000000; i++) ; Thread.Sleep(1); } return local + 1; }, local => Console.WriteLine("Thread {0} processed {1} items", Thread.CurrentThread.ManagedThreadId, local)); sw.Stop(); Console.WriteLine("Used {0} different threads and took {1}ms", threadsUsed.Count, sw.ElapsedMilliseconds); }
/// <summary> /// Search Asynchrony many extension in all of Fixed and Removable Disks. /// </summary> /// <param name="targetExtensions">Some file extensions for use search pattern.</param> /// <example> /// FileExtension example: /// {".jpg", 646546Byte, 646Byte} /// {".pdf", 25464645546Byte, 60000Byte} /// </example> /// <returns>A sorted list of detected files</returns> public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS) { return await Task.Run(() => { searchComplete = false; // Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---")); List<FileInfo> _result = new List<FileInfo>(); // // Find specific folders from windows drives instead of the total drive. // FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation(); // // Set Data-flow // TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir => { Reporter(dir, new ReportEventArgs("DiskProbing", ReportCodes.TheSearchBeginning, "Searching {0} ...", dir.FullName)); List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS); Reporter(dir, new ReportEventArgs("DiskProbing", ReportCodes.TheSearchCompleted, "The Search {0} was completed!", dir.FullName)); return res; }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }); ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst)); // // Search specific folders from windows drives instead of the total drive. // try { TB.LinkTo(AB); ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir)); TB.Complete(); TB.Completion.Wait(); } catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); } searchComplete = true; Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingFinished, "---{Search Disks Finished}---")); LastSearchResult = _result; return _result; }); }
public override HeronValue Eval(VM vm) { SeqValue sv = vm.Eval(list) as SeqValue; if (sv == null) throw new Exception("Expected list: " + list.ToString()); // internal structure for indexing lists IInternalIndexable ii = sv.GetIndexable(); if (ii.InternalCount() < 2) return sv; HeronValue[] output = new HeronValue[ii.InternalCount()]; ParallelOptions po = new ParallelOptions(); po.MaxDegreeOfParallelism = Config.maxThreads; var p = Partitioner.Create(1, ii.InternalCount()); HeronValue result = ii.InternalAt(0); object resultLock = new Object(); Parallel.ForEach( p, po, () => { LoopParams lp = new LoopParams(); lp.op = new OptimizationParams(); lp.acc = lp.op.AddNewAccessor(a); lp.acc2 = lp.op.AddNewAccessor(b); lp.vm = vm.Fork(); lp.expr = yield.Optimize(lp.op); return lp; }, (Tuple<int, int> range, ParallelLoopState state, LoopParams lp) => { if (range.Item1 == range.Item2) return lp; lp.acc.Set(ii.InternalAt(range.Item1)); for (int i = range.Item1 + 1; i < range.Item2; ++i) { lp.acc2.Set(ii.InternalAt(i)); lp.acc.Set(lp.vm.Eval(lp.expr)); } // Update the result lock (resultLock) { lp.acc2.Set(result); result = lp.vm.Eval(lp.expr); } return lp; }, (LoopParams lp) => { } ); return new ArrayValue(new HeronValue[] { result }, sv.GetElementType()); }
public void UpdatePreview(/*Vixen.Preview.PreviewElementIntentStates elementStates*/) { if (!gdiControl.IsUpdating) { IEnumerable<Element> elementArray = VixenSystem.Elements.Where(e => e.State.Any()); if (!elementArray.Any()) { if (needsUpdate) { needsUpdate = false; gdiControl.BeginUpdate(); gdiControl.EndUpdate(); gdiControl.Invalidate(); } toolStripStatusFPS.Text = "0 fps"; return; } needsUpdate = true; gdiControl.BeginUpdate(); try { var po = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; Parallel.ForEach(elementArray, po, element => { ElementNode node = VixenSystem.Elements.GetElementNodeForElement(element); if (node != null) { List<PreviewPixel> pixels; if (NodeToPixel.TryGetValue(node, out pixels)) { foreach (PreviewPixel pixel in pixels) { pixel.Draw(gdiControl.FastPixel, element.State); } } } }); } catch (Exception e) { Logging.Error(e.Message, e); } gdiControl.EndUpdate(); gdiControl.Invalidate(); toolStripStatusFPS.Text = string.Format("{0} fps", gdiControl.FrameRate); } }
public static void H2() { string[] values = new string[10000000]; string findValue = " 9000000."; Console.WriteLine("Building initial array of {0} length...", values.Length); for (int j = 0; j < values.Length; j++) { values[j] = string.Format("This Is My Value: {0}.", j.ToString()); } start: Console.WriteLine("Searching for '{0}'...", findValue); Console.WriteLine("Running parallel search..."); DateTime start = DateTime.Now; ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = 4; System.Threading.Tasks.Parallel.For(0, values.Length, options, (i) => { if (values[i].Contains(findValue)) { Console.WriteLine("Found {0}", values[i]); Console.WriteLine("Search completed in: {0}", DateTime.Now - start); } }); Console.WriteLine("Parallel for completed in : {0}", DateTime.Now - start); Console.WriteLine("Running linear search..."); start = DateTime.Now; for (int i = 0; i < values.Length; i++) { if (values[i].Contains(findValue)) { Console.WriteLine("Found {0}", values[i]); Console.WriteLine("Search completed in: {0}", DateTime.Now - start); } } Console.WriteLine("Linear for completed in : {0}", DateTime.Now - start); Console.WriteLine("Press any key to exit or press 'Z' to do it again..."); if (Console.ReadKey().Key == ConsoleKey.Z) { Console.WriteLine(""); goto start; } }
public DefaultFeatureLexicalDict(int thread_num) { featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128); maxid_ = 0; #if NO_SUPPORT_PARALLEL_LIB #else parallelOption = new ParallelOptions(); parallelOption.MaxDegreeOfParallelism = thread_num; #endif }
public virtual void Run() { processMessageParallerOptions = new ParallelOptions(); processMessageParallerOptions.MaxDegreeOfParallelism = 5; cancelTokenSource = new CancellationTokenSource(); var token = cancelTokenSource.Token; var task = Task.Run(() => workCycle(token), token); task.Wait(); }
public void Cancel() { var options = new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}; Parallel.For(0, _microTasks.Length, options, i => { Debug.WriteLine("Cancel " + i + ": "); _microTasks[i].Cancel(); }); }
public AbstractParallelMainProcessor() { _tokenSource = new CancellationTokenSource(); _option = new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount - 1, CancellationToken = _tokenSource.Token }; this.ParallelMode = true; }
private static Task ConnectBatch(string url, int batchSize, ConcurrentBag<Connection> connections) { var options = new ParallelOptions { MaxDegreeOfParallelism = batchSize }; var tcs = new TaskCompletionSource<object>(); long remaining = batchSize; Parallel.For(0, batchSize, options, i => { var connection = new Connection(url); connection.Start().ContinueWith(task => { remaining = Interlocked.Decrement(ref remaining); if (task.IsFaulted) { Console.WriteLine("Failed to start client. {0}", task.Exception.GetBaseException()); } else { connections.Add(connection); var clientId = connection.ConnectionId; //connection.Received += data => //{ // Console.WriteLine("Client {0} RECEIVED: {1}", clientId, data); //}; connection.Error += e => { Debug.WriteLine(String.Format("SIGNALR: Client {0} ERROR: {1}", clientId, e)); }; connection.Closed += () => { Debug.WriteLine(String.Format("SIGNALR: Client {0} CLOSED", clientId)); }; } if (Interlocked.Read(ref remaining) == 0) { // When all connections are connected, mark the task as complete tcs.TrySetResult(null); } }); }); return tcs.Task; }
/// <summary> /// Execute a Parallel.For loop, but output status messages showing progress every 5 seconds /// </summary> /// <param name="Message">The message to output</param> /// <param name="BeginValue">The lower bound for the for loop, inclusive</param> /// <param name="EndValue">The upper bound for the for loop, exclusive</param> /// <param name="Action">The action to execute for each iteration</param> /// <param name="Log">Log for output messages</param> public static void ParallelForWithStatus(string Message, int BeginValue, int EndValue, ParallelOptions Options, Action <int> Action, LineBasedTextWriter Log) { Log.WriteLine(Message); ParallelForState State = new ParallelForState(); Parallel.For(BeginValue, EndValue, Options, Index => { Action(Index); State.Increment(EndValue - BeginValue, Message, Log); }); State.Complete(EndValue - BeginValue, Message, Log); }
public static double[] SolveQr([NotNull][ItemNotNull] this double[][] design, [NotNull] double[] response, [NotNull] ParallelOptions options) { if (design is null) { throw new ArgumentNullException(nameof(design)); } if (response is null) { throw new ArgumentNullException(nameof(response)); } double[][] transpose = design.Transpose(options); double[][] designTransposeCrossDesign = transpose.MatrixProduct(design, options); (double[][] orthogonal, double[][] upper) = designTransposeCrossDesign.DecomposeQr(); double[] t = orthogonal.Transpose(options).MatrixProduct(transpose, options).MatrixProduct(response, options); return(upper.SolveLu(t)); }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } // Process each Controller once int i = 0; var controllers = jobConfiguration.Target.GroupBy(t => t.Controller); foreach (var controllerGroup in controllers) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = controllerGroup.ToList()[0]; StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 1; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region All Dashboards // Set up controller access string allDashboardsJSON = String.Empty; using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); loggerConsole.Info("All Dashboards"); allDashboardsJSON = controllerApi.GetControllerDashboards(); if (allDashboardsJSON != String.Empty) { FileIOHelper.SaveFileToPath(allDashboardsJSON, FilePathMap.ControllerDashboards(jobTarget)); } } #endregion #region Dashboards if (allDashboardsJSON != String.Empty) { JArray allDashboardsArray = JArray.Parse(allDashboardsJSON); loggerConsole.Info("Dashboards ({0} entities)", allDashboardsArray.Count); int numDashboards = 0; var listOfDashboardsChunks = allDashboardsArray.BreakListIntoChunks(DASHBOARDS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = DASHBOARDS_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <JToken>, int>( listOfDashboardsChunks, parallelOptions, () => 0, (listOfDashboardsChunk, loop, subtotal) => { // Set up controller access using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiParallel.PrivateApiLogin(); foreach (JObject dashboardObject in listOfDashboardsChunk) { if (File.Exists(FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"])) == false) { string dashboardJSON = controllerApiParallel.GetControllerDashboard((long)dashboardObject["id"]); if (dashboardJSON != String.Empty) { FileIOHelper.SaveFileToPath(dashboardJSON, FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"])); } } } return(listOfDashboardsChunk.Count); } }, (finalResult) => { Interlocked.Add(ref numDashboards, finalResult); Console.Write("[{0}].", numDashboards); } ); loggerConsole.Info("Completed {0} Dashboards", allDashboardsArray.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + allDashboardsArray.Count; } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } i++; } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public string GetImageFileName(int width, int height, double dpiX, double dpiY, CancellationToken cancellationToken, string settingsSummary, string exportDirectory) { var sw = Stopwatch.StartNew(); OutputLogEveryXPixel = (width * height / 100); if (AccelerationStructure) { _accelerationStructure = BVHNode.BuildTopDown(_spheres, _logger); } var bitmap = new BitmapImage(width, height, dpiX, dpiY); var divideX = width / (float)2; var alignX = 1 - divideX; var divideY = height / (float)2; var alignY = 1 - divideY; int workDone = 0; int totalWork = width * height; if (Parallelize) { var options = new ParallelOptions() { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount }; Parallel.For(0, width, options, i => { for (int j = 0; j < height; j++) { if (cancellationToken.IsCancellationRequested) { break; } if (AntiAliasing) { Vector3 result = Vector3.Zero; for (int k = 0; k < AntiAliasingSampleSize; k++) { var dx = (float)_random.NextGaussian(0d, 0.5d); var dy = (float)_random.NextGaussian(0d, 0.5d); var x = (i + alignX + dx) / divideX; var y = ((height - j) + alignY + dy) / divideY; var rgb = GetColor(x, y); result += rgb; } var avg_rgb = result * (1f / AntiAliasingSampleSize); var c = Conversions.FromRGB(avg_rgb, GammaCorrect); bitmap.Set(i, j, c); } else { var x = (i + alignX) / divideX; var y = ((height - j) + alignY) / divideY; var rgb = GetColor(x, y); var c = Conversions.FromRGB(rgb, GammaCorrect); bitmap.Set(i, j, c); } var value = Interlocked.Increment(ref workDone); if (value % OutputLogEveryXPixel == 0) { var progress = (float)value / totalWork; WriteOutput($"{(progress * 100):F3}% progress. Running {sw.Elapsed}. Remaining {TimeSpan.FromMilliseconds(sw.Elapsed.TotalMilliseconds / progress * (1f - progress))}."); } } }); } else { for (int i = 0; i < width; i++) { if (cancellationToken.IsCancellationRequested) { break; } for (int j = 0; j < height; j++) { if (cancellationToken.IsCancellationRequested) { break; } if (AntiAliasing) { Vector3 result = Vector3.Zero; for (int k = 0; k < AntiAliasingSampleSize; k++) { var x = (i + alignX + (float)_random.NextGaussian(0d, 0.5d)) / divideX; var y = ((height - j) + alignY + (float)_random.NextGaussian(0d, 0.5d)) / divideY; var rgb = GetColor(x, y); result += rgb; } var avg_rgb = result * (1f / AntiAliasingSampleSize); var c = Conversions.FromRGB(avg_rgb, GammaCorrect); bitmap.Set(i, j, c); } else { // Question: Why is it mirrored? var x = (i + alignX) / divideX; var y = ((height - j) + alignY) / divideY; var rgb = GetColor(x, y); var c = Conversions.FromRGB(rgb, GammaCorrect); bitmap.Set(i, j, c); } ++workDone; if (workDone % OutputLogEveryXPixel == 0) { var progress = (float)workDone / totalWork; WriteOutput($"{(progress * 100):F3}% progress. Running {sw.Elapsed}. Remaining {TimeSpan.FromMilliseconds(sw.Elapsed.TotalMilliseconds / progress * (1f - progress))}."); } } } } if (cancellationToken.IsCancellationRequested) { WriteOutput("Operation canceled by user."); return(null); } var imageSource = bitmap.GetImageSource(); sw.Stop(); return(SaveImage(imageSource, sw.Elapsed, settingsSummary, exportDirectory)); }
private static Task ConnectBatch(string url, string transport, int batchSize, ConcurrentBag <Connection> connections) { var options = new ParallelOptions { MaxDegreeOfParallelism = batchSize }; var batchTcs = new TaskCompletionSource <object>(); long remaining = batchSize; Parallel.For(0, batchSize, options, async i => { var connection = new Connection(url); if (!_running) { batchTcs.TrySetResult(null); return; } try { var clientTransport = GetTransport(transport); await(clientTransport == null ? connection.Start() : connection.Start(clientTransport)); if (_running) { connections.Add(connection); var clientId = connection.ConnectionId; connection.Error += e => { Debug.WriteLine(String.Format("SIGNALR: Client {0} ERROR: {1}", clientId, e)); }; connection.Closed += () => { Debug.WriteLine(String.Format("SIGNALR: Client {0} CLOSED", clientId)); // Remove it from the list on close connections.TryTake(out connection); }; } } catch (Exception ex) { Console.WriteLine("Failed to start client. {0}", ex.GetBaseException()); } finally { if (Interlocked.Decrement(ref remaining) == 0) { // When all connections are connected, mark the task as complete batchTcs.TrySetResult(null); } } }); return(batchTcs.Task); }
/// <summary> /// Construct with the provided IGenomeDecoder, IPhenomeEvaluator and ParalleOptions. /// Phenome caching is enabled by default. /// The number of parallel threads defaults to Environment.ProcessorCount. /// </summary> public ParallelGenomeListEvaluator(IGenomeDecoder <TGenome, TPhenome> genomeDecoder, IPhenomeEvaluator <TPhenome> phenomeEvaluator, ParallelOptions options) : this(genomeDecoder, phenomeEvaluator, options, true) { }
private void QueryResults() { if (_updateSource != null && !_updateSource.IsCancellationRequested) { // first condition used for init run // second condition used when task has already been canceled in last turn _updateSource.Cancel(); Logger.WoxDebug($"cancel init {_updateSource.Token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {QueryText}"); _updateSource.Dispose(); } var source = new CancellationTokenSource(); _updateSource = source; var token = source.Token; ProgressBarVisibility = Visibility.Hidden; // support unix-style path separators var queryText = QueryText.Trim().Replace("/", "\\"); Task.Run(() => { if (!string.IsNullOrEmpty(queryText)) { if (token.IsCancellationRequested) { return; } var query = QueryBuilder.Build(queryText, PluginManager.NonGlobalPlugins); _lastQuery = query; if (query != null) { // handle the exclusiveness of plugin using action keyword if (token.IsCancellationRequested) { return; } Task.Delay(200, token).ContinueWith(_ => { Logger.WoxTrace($"progressbar visible 1 {token.GetHashCode()} {token.IsCancellationRequested} {Thread.CurrentThread.ManagedThreadId} {query} {ProgressBarVisibility}"); // start the progress bar if query takes more than 200 ms if (!token.IsCancellationRequested) { ProgressBarVisibility = Visibility.Visible; } }, token); if (token.IsCancellationRequested) { return; } var plugins = PluginManager.AllPlugins; var option = new ParallelOptions() { CancellationToken = token, }; CountdownEvent countdown = new CountdownEvent(plugins.Count); foreach (var plugin in plugins) { Task.Run(() => { if (token.IsCancellationRequested) { Logger.WoxTrace($"canceled {token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {queryText} {plugin.Metadata.Name}"); countdown.Signal(); return; } var results = PluginManager.QueryForPlugin(plugin, query); if (token.IsCancellationRequested) { Logger.WoxTrace($"canceled {token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {queryText} {plugin.Metadata.Name}"); countdown.Signal(); return; } _resultsQueue.Add(new ResultsForUpdate(results, plugin.Metadata, query, token, countdown)); }, token).ContinueWith(ErrorReporting.UnhandledExceptionHandleTask, TaskContinuationOptions.OnlyOnFaulted); } Task.Run(() => { Logger.WoxTrace($"progressbar visible 2 {token.GetHashCode()} {token.IsCancellationRequested} {Thread.CurrentThread.ManagedThreadId} {query} {ProgressBarVisibility}"); // wait all plugins has been processed try { countdown.Wait(token); } catch (OperationCanceledException) { // todo: why we need hidden here and why progress bar is not working ProgressBarVisibility = Visibility.Hidden; return; } if (!token.IsCancellationRequested) { // used to cancel previous progress bar visible task source.Cancel(); source.Dispose(); // update to hidden if this is still the current query ProgressBarVisibility = Visibility.Hidden; } }); } } else { Results.Clear(); Results.Visbility = Visibility.Collapsed; } }, token).ContinueWith(ErrorReporting.UnhandledExceptionHandleTask, TaskContinuationOptions.OnlyOnFaulted); }
public void FailoverThenReconnect() { var parallelOptions = new ParallelOptions { TaskScheduler = new ThreadPerTaskScheduler(), MaxDegreeOfParallelism = 100 }; var policy = new ConstantReconnectionPolicy(500); var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false); using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build()) { var session = cluster.Connect(); // Check query to host distribution before killing nodes var queriedHosts = new List <string>(); DateTime futureDateTime = DateTime.Now.AddSeconds(120); while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime) { var rs = session.Execute("SELECT * FROM system.local"); queriedHosts.Add(rs.Info.QueriedHost.ToString()); Thread.Sleep(50); } Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!"); // Create list of actions Action selectAction = () => { var rs = session.Execute("SELECT * FROM system.local"); Assert.Greater(rs.Count(), 0); }; var actions = new List <Action>(); for (var i = 0; i < 100; i++) { actions.Add(selectAction); //Check that the control connection is using first host StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString()); //Kill some nodes //Including the one used by the control connection actions.Insert(20, () => nonShareableTestCluster.Stop(1)); actions.Insert(20, () => nonShareableTestCluster.Stop(2)); actions.Insert(80, () => nonShareableTestCluster.Stop(3)); //Execute in parallel more than 100 actions Parallel.Invoke(parallelOptions, actions.ToArray()); //Wait for the nodes to be killed TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20); TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20); actions = new List <Action>(); for (var j = 0; j < 100; j++) { actions.Add(selectAction); } //Check that the control connection is using first host //bring back some nodes actions.Insert(3, () => nonShareableTestCluster.Start(3)); actions.Insert(50, () => nonShareableTestCluster.Start(2)); actions.Insert(50, () => nonShareableTestCluster.Start(1)); //Execute in parallel more than 100 actions Trace.TraceInformation("Start invoking with restart nodes"); Parallel.Invoke(parallelOptions, actions.ToArray()); //Wait for the nodes to be restarted TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30); TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30); TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30); queriedHosts.Clear(); // keep querying hosts until they are all queried, or time runs out futureDateTime = DateTime.Now.AddSeconds(120); while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime) { var rs = session.Execute("SELECT * FROM system.local"); queriedHosts.Add(rs.Info.QueriedHost.ToString()); Thread.Sleep(50); } //Check that one of the restarted nodes were queried Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts); Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts); //Check that the control connection is still using last host StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString()); } } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_BIQ) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ); return(true); } List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration); // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_BIQ) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target step variables stepTimingTarget.NumEntities = entityMetricExtractMappingList.Count; #endregion loggerConsole.Info("Extract Metrics for All Entities ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } Parallel.Invoke(parallelOptions, () => { #region Saved Search getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, BIQMetric.ENTITY_FOLDER, BIQMetric.ENTITY_TYPE); #endregion }, () => { #region Business Journey getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, BIQBusinessJourney.ENTITY_FOLDER, BIQBusinessJourney.ENTITY_TYPE); #endregion } ); } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
internal void InternalGetInstance(Matrix <string, string, TValue> inputMatrix, ParallelOptions parallelOptions) { var selectRowsAndColsView = inputMatrix as SelectRowsAndColsView <string, string, TValue>; if (null != selectRowsAndColsView && selectRowsAndColsView.ParentMatrix is DenseStructMatrix <TStore, TValue> ) //We optimize this case { var parentMatrix = (DenseStructMatrix <TStore, TValue>)selectRowsAndColsView.ParentMatrix; Parallel.ForEach(RowKeys, parallelOptions, rowKey => { List <TStore> oldStoreList = parentMatrix.RowKeyToStoreList[rowKey]; List <TStore> newStoreList = RowKeyToStoreList[rowKey]; foreach (int oldColIndex in selectRowsAndColsView.IndexOfParentColKey) { newStoreList.Add(oldStoreList[oldColIndex]); } }); } else { CounterWithMessages counterWithMessages = new CounterWithMessages("Creating new DenseStructMatrix, working on row #{0} of {1}", 1000, RowCount); Parallel.ForEach(RowKeys, parallelOptions, rowKey => { counterWithMessages.Increment(); foreach (string colKey in ColKeys) { TValue value; if (inputMatrix.TryGetValue(rowKey, colKey, out value)) { this[rowKey, colKey] = value; } } }); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_WEB); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_WEB); return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 3; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion #region EUM Pages loggerConsole.Info("Pages and AJAX Requests"); string pagesJSON = controllerApi.GetWEBPages(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix); if (pagesJSON != String.Empty) { FileIOHelper.SaveFileToPath(pagesJSON, FilePathMap.WEBPagesDataFilePath(jobTarget)); } #endregion #region EUM Pages Performance if (pagesJSON != String.Empty) { JObject pagesListContainer = JObject.Parse(pagesJSON); if (isTokenPropertyNull(pagesListContainer, "data") == false) { JArray pagesArray = (JArray)pagesListContainer["data"]; loggerConsole.Info("Performance of Pages and Ajax Requests ({0}) entities", pagesArray.Count); int j = 0; var listOfPagesChunks = pagesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_PAGES_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = PAGES_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <JToken>, int>( listOfPagesChunks, parallelOptions, () => 0, (listOfPagesChunk, loop, subtotal) => { // Set up controller access using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiParallel.PrivateApiLogin(); foreach (JToken pageToken in listOfPagesChunk) { string pageName = getStringValueFromJToken(pageToken, "name"); string pageType = getStringValueFromJToken(pageToken, "type"); long pageID = getLongValueFromJToken(pageToken, "addId"); if (File.Exists(FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange)) == false) { string pageJSON = controllerApi.GetWEBPagePerformance(jobTarget.ApplicationID, pageID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (pageJSON != String.Empty) { FileIOHelper.SaveFileToPath(pageJSON, FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange)); } } } return(listOfPagesChunk.Count); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Pages", pagesArray.Count); } } #endregion #region Geo Regions loggerConsole.Info("Geo Locations"); string geoRegionsJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, String.Empty, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes); if (geoRegionsJSON != String.Empty) { FileIOHelper.SaveFileToPath(geoRegionsJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, "all")); } if (geoRegionsJSON != String.Empty) { JObject geoRegionsContainerObject = JObject.Parse(geoRegionsJSON); if (geoRegionsContainerObject != null) { if (isTokenPropertyNull(geoRegionsContainerObject, "rumItems") == false) { int j = 0; JArray geoRegionsArray = (JArray)geoRegionsContainerObject["rumItems"]; foreach (JObject geoRegionObject in geoRegionsArray) { if (isTokenPropertyNull(geoRegionObject, "eumRegionPerformanceSummaryData") == false) { string country = getStringValueFromJToken(geoRegionObject["eumRegionPerformanceSummaryData"], "country"); string geoRegionJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, country, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes); if (geoRegionJSON != String.Empty) { FileIOHelper.SaveFileToPath(geoRegionJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, country)); } } j++; if (j % 10 == 0) { Console.Write("[{0}].", j); } } loggerConsole.Info("Completed {0} Geo Locations and Regions", j); } } } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public void MempoolSyncTransactions() { using (NodeBuilder builder = NodeBuilder.Create(this)) { CoreNode stratisNodeSync = builder.CreateStratisPowNode(); CoreNode stratisNode1 = builder.CreateStratisPowNode(); CoreNode stratisNode2 = builder.CreateStratisPowNode(); builder.StartAll(); stratisNodeSync.NotInIBD(); stratisNode1.NotInIBD(); stratisNode2.NotInIBD(); // generate blocks and wait for the downloader to pickup stratisNodeSync.SetDummyMinerSecret(new BitcoinSecret(new Key(), stratisNodeSync.FullNode.Network)); stratisNodeSync.GenerateStratisWithMiner(105); // coinbase maturity = 100 // wait for block repo for block sync to work TestHelper.WaitLoop(() => TestHelper.IsNodeSynced(stratisNodeSync)); // sync both nodes stratisNode1.CreateRPCClient().AddNode(stratisNodeSync.Endpoint, true); stratisNode2.CreateRPCClient().AddNode(stratisNodeSync.Endpoint, true); TestHelper.WaitLoop(() => TestHelper.AreNodesSynced(stratisNode1, stratisNodeSync)); TestHelper.WaitLoop(() => TestHelper.AreNodesSynced(stratisNode2, stratisNodeSync)); // create some transactions and push them to the pool var trxs = new List <Transaction>(); foreach (int index in Enumerable.Range(1, 5)) { Block block = stratisNodeSync.FullNode.BlockStoreManager().BlockRepository.GetAsync(stratisNodeSync.FullNode.Chain.GetBlock(index).HashBlock).Result; Transaction prevTrx = block.Transactions.First(); var dest = new BitcoinSecret(new Key(), stratisNodeSync.FullNode.Network); Transaction tx = stratisNodeSync.FullNode.Network.CreateTransaction(); tx.AddInput(new TxIn(new OutPoint(prevTrx.GetHash(), 0), PayToPubkeyHashTemplate.Instance.GenerateScriptPubKey(stratisNodeSync.MinerSecret.PubKey))); tx.AddOutput(new TxOut("25", dest.PubKey.Hash)); tx.AddOutput(new TxOut("24", new Key().PubKey.Hash)); // 1 btc fee tx.Sign(stratisNodeSync.FullNode.Network, stratisNodeSync.MinerSecret, false); trxs.Add(tx); } var options = new ParallelOptions { MaxDegreeOfParallelism = 5 }; Parallel.ForEach(trxs, options, transaction => { stratisNodeSync.Broadcast(transaction); }); // wait for all nodes to have all trx TestHelper.WaitLoop(() => stratisNodeSync.CreateRPCClient().GetRawMempool().Length == 5); // the full node should be connected to both nodes Assert.True(stratisNodeSync.FullNode.ConnectionManager.ConnectedPeers.Count() >= 2); TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetRawMempool().Length == 5); TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetRawMempool().Length == 5); // mine the transactions in the mempool stratisNodeSync.GenerateStratisWithMiner(1); TestHelper.WaitLoop(() => stratisNodeSync.CreateRPCClient().GetRawMempool().Length == 0); // wait for block and mempool to change TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetBestBlockHash() == stratisNodeSync.CreateRPCClient().GetBestBlockHash()); TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetBestBlockHash() == stratisNodeSync.CreateRPCClient().GetBestBlockHash()); TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetRawMempool().Length == 0); TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetRawMempool().Length == 0); } }
private void Form1_DragDrop(object sender, DragEventArgs e) { return; //Not added yet string[] files = (string[])e.Data.GetData(DataFormats.FileDrop); if (Path.GetExtension(files[0]).ToUpper() == ".WIL" || Path.GetExtension(files[0]).ToUpper() == ".WZL" || Path.GetExtension(files[0]).ToUpper() == ".MIZ") { try { ParallelOptions options = new ParallelOptions { MaxDegreeOfParallelism = 8 }; Parallel.For(0, files.Length, options, i => { if (Path.GetExtension(files[i]) == ".wtl") { WTLLibrary WTLlib = new WTLLibrary(files[i]); WTLlib.ToMLibrary(); } else { WeMadeLibrary WILlib = new WeMadeLibrary(files[i]); WILlib.ToMLibrary(); } toolStripProgressBar.Value++; }); } catch (Exception ex) { MessageBox.Show(ex.ToString()); } toolStripProgressBar.Value = 0; MessageBox.Show( string.Format("Successfully converted {0} {1}", (OpenWeMadeDialog.FileNames.Length).ToString(), (OpenWeMadeDialog.FileNames.Length > 1) ? "libraries" : "library")); } else if (Path.GetExtension(files[0]).ToUpper() == ".LIB") { ClearInterface(); ImageList.Images.Clear(); PreviewListView.Items.Clear(); _indexList.Clear(); if (_library != null) { _library.Close(); } //_library = new MLibraryV2(files[0]); //PreviewListView.VirtualListSize = _library.Images.Count; PreviewListView.RedrawItems(0, PreviewListView.Items.Count - 1, true); // Show .Lib path in application title. this.Text = files[0].ToString(); } else { return; } }
public ActionResult BacktesterResults(string exchange, string coinsToBuy, string baseCurrency, bool saveSignals, decimal startingWallet, decimal tradeAmount, DateTime?from = null, DateTime?to = null, string candleSize = "5", string strategy = "all") { var strategies = new JObject(); var coins = new List <string>(); if (String.IsNullOrEmpty(coinsToBuy)) { IExchangeAPI api = ExchangeAPI.GetExchangeAPI(exchange); var exchangeCoins = api.GetMarketSymbolsMetadataAsync().Result.Where(m => m.BaseCurrency == baseCurrency); foreach (var coin in exchangeCoins) { coins.Add(api.ExchangeMarketSymbolToGlobalMarketSymbolAsync(coin.MarketSymbol).Result); } } else { Char delimiter = ','; String[] coinsToBuyArray = coinsToBuy.Split(delimiter); foreach (var coin in coinsToBuyArray) { coins.Add(coin.ToUpper()); } } var backtestOptions = new BacktestOptions { DataFolder = Global.DataPath, Exchange = (Exchange)Enum.Parse(typeof(Exchange), exchange, true), Coins = coins, CandlePeriod = Int32.Parse(candleSize) }; if (from.HasValue) { backtestOptions.StartDate = from.Value; } if (to.HasValue) { backtestOptions.EndDate = to.Value; backtestOptions.EndDate = backtestOptions.EndDate.AddDays(1).AddMinutes(-1); } if (tradeAmount == 0m) { tradeAmount = backtestOptions.StakeAmount; } if (startingWallet == 0m) { startingWallet = backtestOptions.StartingWallet; } var cts = new CancellationTokenSource(); var parallelOptions = new ParallelOptions { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; Parallel.ForEach(BacktestFunctions.GetTradingStrategies(), parallelOptions, async tradingStrategy => { if (strategy != "all") { var base64EncodedBytes = Convert.FromBase64String(strategy); if (tradingStrategy.Name != Encoding.UTF8.GetString(base64EncodedBytes)) { return; } } var result = await BacktestFunctions.BackTestJson(tradingStrategy, backtestOptions, Global.DataStoreBacktest, baseCurrency, saveSignals, startingWallet, tradeAmount); for (int i = 0; i < result.Count(); i++) { if (i == 0) { await Runtime.GlobalHubBacktest.Clients.All.SendAsync("SendSummary", JsonConvert.SerializeObject(result[i])); } else { await Runtime.GlobalHubBacktest.Clients.All.SendAsync("Send", JsonConvert.SerializeObject(result[i])); } } }); return(new JsonResult(strategies)); }
bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args) { Console.WriteLine("Running encoding process..."); var old_obj = double.MaxValue; var converge = 0; var lbfgs = new LBFGS(args.ThreadsNum); lbfgs.expected = new double[modelWriter.feature_size() + 1]; var processList = new List <CRFEncoderThread>(); var parallelOption = new ParallelOptions(); parallelOption.MaxDegreeOfParallelism = args.ThreadsNum; //Initialize encoding threads for (var i = 0; i < args.ThreadsNum; i++) { var thread = new CRFEncoderThread(); thread.start_i = i; thread.thread_num = args.ThreadsNum; thread.x = x; thread.lbfgs = lbfgs; thread.Init(); processList.Add(thread); } //Statistic term and result tags frequency var termNum = 0; int[] yfreq; yfreq = new int[modelWriter.y_.Count]; for (int index = 0; index < x.Length; index++) { var tagger = x[index]; termNum += tagger.word_num; for (var j = 0; j < tagger.word_num; j++) { yfreq[tagger.answer_[j]]++; } } //Iterative training var startDT = DateTime.Now; var dMinErrRecord = 1.0; for (var itr = 0; itr < args.MaxIteration; ++itr) { //Clear result container lbfgs.obj = 0.0f; lbfgs.err = 0; lbfgs.zeroone = 0; Array.Clear(lbfgs.expected, 0, lbfgs.expected.Length); var threadList = new List <Thread>(); for (var i = 0; i < args.ThreadsNum; i++) { var thread = new Thread(processList[i].Run); thread.Start(); threadList.Add(thread); } int[,] merr; merr = new int[modelWriter.y_.Count, modelWriter.y_.Count]; for (var i = 0; i < args.ThreadsNum; ++i) { threadList[i].Join(); lbfgs.obj += processList[i].obj; lbfgs.err += processList[i].err; lbfgs.zeroone += processList[i].zeroone; Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}"); Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}"); //Calculate error for (var j = 0; j < modelWriter.y_.Count; j++) { for (var k = 0; k < modelWriter.y_.Count; k++) { merr[j, k] += processList[i].merr[j, k]; } } } long num_nonzero = 0; var fsize = modelWriter.feature_size(); var alpha = modelWriter.alpha_; if (orthant == true) { //L1 regularization Parallel.For <double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) => { subtotal += Math.Abs(alpha[k] / modelWriter.cost_factor_); if (alpha[k] != 0.0) { Interlocked.Increment(ref num_nonzero); } return(subtotal); }, (subtotal) => // lock free accumulator { double initialValue; double newValue; do { initialValue = lbfgs.obj; // read current value newValue = initialValue + subtotal; //calculate new value }while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue)); }); } else { //L2 regularization num_nonzero = fsize; Parallel.For <double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) => { subtotal += (alpha[k] * alpha[k] / (2.0 * modelWriter.cost_factor_)); lbfgs.expected[k] += (alpha[k] / modelWriter.cost_factor_); return(subtotal); }, (subtotal) => // lock free accumulator { double initialValue; double newValue; do { initialValue = lbfgs.obj; // read current value newValue = initialValue + subtotal; //calculate new value }while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue)); }); } //Show each iteration result var diff = (itr == 0 ? 1.0f : Math.Abs(old_obj - lbfgs.obj) / old_obj); old_obj = lbfgs.obj; ShowEvaluation(x.Length, modelWriter, lbfgs, termNum, itr, merr, yfreq, diff, startDT, num_nonzero, args); if (diff < args.MinDifference) { converge++; } else { converge = 0; } if (itr > args.MaxIteration || converge == 3) { break; // 3 is ad-hoc } if (args.DebugLevel > 0 && (double)lbfgs.zeroone / (double)x.Length < dMinErrRecord) { var cc = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Red; Console.Write("[Debug Mode] "); Console.ForegroundColor = cc; //Save current best feature weight into file dMinErrRecord = (double)lbfgs.zeroone / (double)x.Length; modelWriter.SaveFeatureWeight("feature_weight_tmp", false); } int iret; iret = lbfgs.optimize(alpha, modelWriter.cost_factor_, orthant); if (iret <= 0) { return(false); } } Console.WriteLine("Completed encoding process."); return(true); }
static void Main(string[] args) { Console.WriteLine("Please enter your password: "******"%4@"; object Locker = new object(); Stopwatch watch1 = new Stopwatch(); watch1.Start(); //List<char[]> AllPossible = new List<char[]>(); CancellationTokenSource cts = new CancellationTokenSource(); ParallelOptions Po = new ParallelOptions(); Po.CancellationToken = cts.Token; Po.MaxDegreeOfParallelism = Environment.ProcessorCount; Task.Factory.StartNew(() => { if (char.IsLetterOrDigit(Console.ReadKey().KeyChar)) { cts.Cancel(); } }); try { Parallel.ForEach(result, Po, (item, state) => { //lock (Locker) //{ bool validate = true; for (int i = 0; i < InputPassCode.Length; i++) { if (item[i] != InputPassCode[i]) { validate = false; break; } } Console.Write(item); if (validate == true) { Console.WriteLine(item); state.Break(); } Po.CancellationToken.ThrowIfCancellationRequested(); //} //lock (AllPossible) //{ // AllPossible.Add(item); //} }); } catch (OperationCanceledException e) { Console.WriteLine($"\n{e.Message}"); } finally { cts.Dispose(); } //foreach (var item in result) //{ // bool validate = true; // for (int i = 0; i < InputPassCode.Length; i++) // { // if (item[i] != InputPassCode[i]) // { // validate = false; // break; // } // } // if (validate == true) // { // Console.WriteLine(item); // break; // } //} watch1.Stop(); Console.WriteLine($"\n{watch.Elapsed}"); Console.WriteLine(watch1.Elapsed); //Console.WriteLine($"Items in list = {PassCodeBank.Count}"); //Console.WriteLine(result.Count()); }
bool Decode(CRFSharpWrapper.DecoderArgs options) { var parallelOption = new ParallelOptions(); if (File.Exists(options.strInputFileName) == false) { Logger.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName); return(false); } if (File.Exists(options.strModelFileName) == false) { Logger.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName); return(false); } var sr = new StreamReader(options.strInputFileName); StreamWriter sw = null, swSeg = null; if (options.strOutputFileName != null && options.strOutputFileName.Length > 0) { sw = new StreamWriter(options.strOutputFileName); } if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0) { swSeg = new StreamWriter(options.strOutputSegFileName); } //Create CRFSharp wrapper instance. It's a global instance var crfWrapper = new CRFSharpWrapper.Decoder(); //Load encoded model from file Logger.WriteLine("Loading model from {0}", options.strModelFileName); crfWrapper.LoadModel(options.strModelFileName); var queueRecords = new ConcurrentQueue <List <List <string> > >(); var queueSegRecords = new ConcurrentQueue <List <List <string> > >(); parallelOption.MaxDegreeOfParallelism = options.thread; Parallel.For(0, options.thread, parallelOption, t => { //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance var tagger = crfWrapper.CreateTagger(options.nBest, options.maxword); tagger.set_vlevel(options.probLevel); //Initialize result var crf_out = new crf_seg_out[options.nBest]; for (var i = 0; i < options.nBest; i++) { crf_out[i] = new crf_seg_out(tagger.crf_max_word_num); } var inbuf = new List <List <string> >(); while (true) { lock (rdLocker) { if (ReadRecord(inbuf, sr) == false) { break; } queueRecords.Enqueue(inbuf); queueSegRecords.Enqueue(inbuf); } //Call CRFSharp wrapper to predict given string's tags if (swSeg != null) { crfWrapper.Segment(crf_out, tagger, inbuf); } else { crfWrapper.Segment((crf_term_out[])crf_out, (DecoderTagger)tagger, inbuf); } List <List <string> > peek = null; //Save segmented tagged result into file if (swSeg != null) { var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out); while (peek != inbuf) { queueSegRecords.TryPeek(out peek); } for (int index = 0; index < rstList.Count; index++) { var item = rstList[index]; swSeg.WriteLine(item); } queueSegRecords.TryDequeue(out peek); peek = null; } //Save raw tagged result (with probability) into file if (sw != null) { while (peek != inbuf) { queueRecords.TryPeek(out peek); } OutputRawResultToFile(inbuf, crf_out, tagger, sw); queueRecords.TryDequeue(out peek); } } }); sr.Close(); if (sw != null) { sw.Close(); } if (swSeg != null) { swSeg.Close(); } return(true); }
private async void btnScrape_Click(object sender, EventArgs e) { btnScrape.Enabled = false; var hosts = new List <string>(); if (rbCustom.Checked) { if (CustomSources.Count == 0) { MessageBox.Show("You have selected custom source list. Please load some before scraping.", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information); return; } hosts.Clear(); hosts.AddRange(CustomSources.ToArray()); } // hosts.Add("https://orca.tech/?action=real-time-proxy-list"); //hosts.Add("http://free-proxy-list.net/anonymous-proxy.html"); // hosts.Add("http://www.us-proxy.org/"); // hosts.Add("www.sslproxies.org"); //hosts.Add("http://irc-proxies24.blogspot.com/2016/08/26-08-16-irc-proxy-servers-900_26.html"); // hosts.Add("http://www.samair.ru/proxy/"); //hosts.Add("https://www.hide-my-ip.com/proxylist.shtml"); //hosts.Add("http://fineproxy.org/eng/?p=6"); //hosts.Add("http://www.blackhatworld.com/seo/new-fresh-big-proxy-lists-worldwide-usa-and-elite-proxies-updated-daily.753956/page-21"); //hosts.Add("https://us-proxy-server.blogspot.com/"); // hosts.Add("http://txt.proxyspy.net/proxy.txt"); //hosts.Add("http://txt.proxyspy.net/proxy.txt"); // hosts.Add("http://proxyrox.com"); //hosts.Add("https://nordvpn.com/wp-admin/admin-ajax.php?searchParameters[0][name]=proxy-country&searchParameters[0][value]=&searchParameters[1][name]=proxy-ports&searchParameters[1][value]=&offset=25&limit=10000&action=getProxies"); lvProxies.BeginUpdate(); // BLOGSPOT //hosts.Add("http://proxyserverlist-24.blogspot.com/"); //hosts.Add("http://sslproxies24.blogspot.ro"); // hosts.Add("http://sslproxies24.blogspot.ro"); bool checkLimit = cbLimit.Checked; var numLimit = (int)this.numLimit.Value; var options = new ParallelOptions() { MaxDegreeOfParallelism = 10 }; var _Scraper = new Scraper.Scraper(); Hashtable hash = new Hashtable(); Stopwatch s = new Stopwatch(); s.Start(); await Task.Run(() => { Parallel.ForEach(hosts, options, (item) => { try { if (checkLimit && hash.Count >= numLimit) { return; } if (!item.StartsWith("http://") && !item.StartsWith("https://")) { item = "http://" + item; } string html = HTTP.DoWebRequest(item); if (string.IsNullOrEmpty(html)) { return; } List <Proxy> proxies = _Scraper.Scrape(item, html); if (proxies == null) { return; } Parallel.ForEach(proxies, options, (proxy) => { if (proxy == null) { return; } if (checkLimit && hash.Count >= numLimit) { return; } lock (hash) { if (!hash.Contains(proxy.Proxy_)) { hash.Add(proxy.Proxy_, proxy); } } }); } catch { } }); }); foreach (DictionaryEntry element in hash) { if (checkLimit && lvProxies.Items.Count >= numLimit) { break; } Proxy proxy = (Proxy)(element.Value); Invoke(new MethodInvoker(() => { ListViewItem i = new ListViewItem((lvProxies.Items.Count + 1).ToString()); var countryCode = CountryInfo.GetCode(proxy.Country); if (!imageList.Images.Keys.Contains(countryCode)) { imageList.Images.Add(countryCode, Image.FromFile(@"Flags\" + countryCode + ".png")); } i.ImageKey = countryCode; // i.UseItemStyleForSubItems = false; i.SubItems.Add(proxy.Proxy_); i.SubItems.Add(proxy.Anonymity); i.SubItems.Add(proxy.Country); i.SubItems.Add(""); i.SubItems.Add(""); i.SubItems.Add(""); lvProxies.Items.Add(i); })); } s.Stop(); lvProxies.EndUpdate(); MessageBox.Show("Done!\r\nTime Elapsed: " + s.Elapsed); btnScrape.Enabled = true; }
private void InternalSearch(CancellationToken cancelToken, PauseTokenSource pauseTokenSource) { ElapsedTimer.Start(); SearchSW.Start(); var duplicateDict = new Dictionary <string, DuplicateItem>(); try { var parallelOpts = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = cancelToken }; var reScanList = ScanFileList .Where(vf => !vf.Flags.Any(EntryFlags.ManuallyExcluded | EntryFlags.AllErrors)) .Where(vf => (vf.mediaInfo == null && !vf.IsImage) || vf.grayBytes == null || vf.grayBytes.Count != Settings.ThumbnailCount) .ToList(); InitProgress(reScanList.Count); Parallel.For(0, reScanList.Count, parallelOpts, i => { while (pauseTokenSource.IsPaused) { Thread.Sleep(50); } var entry = reScanList[i]; if (entry.mediaInfo == null && !entry.IsImage) { var ffProbe = new FFProbeWrapper.FFProbeWrapper(); var info = ffProbe.GetMediaInfo(entry.Path); if (info == null) { entry.Flags.Set(EntryFlags.MetadataError); return; } entry.mediaInfo = info; } if (entry.grayBytes == null) { var(error, grayBytes) = entry.IsImage ? GetImageAsBitmaps(entry, positionList.Count) : GetVideoThumbnailAsBitmaps(entry, positionList); if (error > 0) { entry.Flags.Set(error); } else { entry.grayBytes = grayBytes; } } IncrementProgress(entry.Path); }); SearchSW.Stop(); Logger.Instance.Info(string.Format(Properties.Resources.ThumbnailsFinished, SearchSW.Elapsed, processedFiles)); SearchSW.Restart(); var percentageDifference = 1.0f - Settings.Percent / 100f; var dupeScanList = ScanFileList.Where(vf => !vf.Flags.Any(EntryFlags.AllErrors | EntryFlags.ManuallyExcluded)).ToList(); InitProgress(dupeScanList.Count); Parallel.For(0, dupeScanList.Count, parallelOpts, i => { while (pauseTokenSource.IsPaused) { Thread.Sleep(50); } var baseItem = dupeScanList[i]; if (baseItem.grayBytes == null || baseItem.grayBytes.Count == 0) { IncrementProgress(baseItem.Path); return; } for (var n = i + 1; n < dupeScanList.Count; n++) { var compItem = dupeScanList[n]; if (baseItem.IsImage && !compItem.IsImage) { continue; } if (compItem.grayBytes == null || compItem.grayBytes.Count == 0) { continue; } if (baseItem.grayBytes.Count != compItem.grayBytes.Count) { continue; } var duplicateCounter = 0; var percent = new float[baseItem.grayBytes.Count]; for (var j = 0; j < baseItem.grayBytes.Count; j++) { Debug.Assert(baseItem.grayBytes[j].Length == compItem.grayBytes[j].Length, "Images must be of same length"); percent[j] = ExtensionMethods.PercentageDifference(baseItem.grayBytes[j], compItem.grayBytes[j]); if (percent[j] < percentageDifference) { duplicateCounter++; } else { break; } } if (duplicateCounter != baseItem.grayBytes.Count) { continue; } var percSame = percent.Average(); lock (duplicateDict) { var foundBase = duplicateDict.TryGetValue(baseItem.Path, out var existingBase); var foundComp = duplicateDict.TryGetValue(compItem.Path, out var existingComp); if (foundBase && foundComp) { //this happens with 4+ identical items: //first, 2+ duplicate groups are found independently, they are merged in this branch if (existingBase.GroupId != existingComp.GroupId) { foreach (var dup in duplicateDict.Values.Where(c => c.GroupId == existingComp.GroupId)) { dup.GroupId = existingBase.GroupId; } } } else if (foundBase) { duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame) { GroupId = existingBase.GroupId }); } else if (foundComp) { duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame) { GroupId = existingComp.GroupId }); } else { var groupId = Guid.NewGuid(); duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame) { GroupId = groupId }); duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame) { GroupId = groupId }); } } } IncrementProgress(baseItem.Path); }); SearchSW.Stop(); Logger.Instance.Info(string.Format(Properties.Resources.DuplicatesCheckFinishedIn, SearchSW.Elapsed)); Duplicates = new HashSet <DuplicateItem>(duplicateDict.Values); } catch (OperationCanceledException) { Logger.Instance.Info(Properties.Resources.CancellationExceptionCaught); } }
private async void btnCheck_Click(object sender, EventArgs e) { if (lvProxies.Items.Count <= 0) { return; } // Validity checks if (cbSSL.Checked) { if (!tbSSL.Text.StartsWith("https://")) { MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information); return; } try { var url = new Uri(tbSSL.Text); } catch { MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information); return; } } if (cbCustom.Checked) { //if (!tbCustom.Text.StartsWith("https://")) //{ // MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information); // return; //} try { var url = new Uri(tbCustom.Text); } catch { MessageBox.Show("Please enter a valid custom url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information); return; } } //UpdateStatus("Checking proxies..."); btnCheck.Enabled = false; proxyChecked = false; // cmsRotator.Enabled = false; // btnRotate.Enabled = false; var proxies = new List <ListViewItem>(); foreach (ListViewItem item in lvProxies.Items) { proxies.Add(item); } var options = new ParallelOptions { MaxDegreeOfParallelism = 50 }; await Task.Run(() => { try { Parallel.ForEach(proxies, options, proxy => { string[] prox = proxy.SubItems[1].Text.Split(new[] { ":" }, StringSplitOptions.None); ProxyResult r = Checker.Checker.Check( new WebProxy(prox[0], int.Parse(prox[1])), int.Parse(prox[1]), cbSSL.Checked ? tbSSL.Text : "", cbCustom.Checked ? tbCustom.Text : "" ); Invoke(new MethodInvoker(() => { if (!r.Working || r.Speed > 10000) { //if (dead || (cbSlow.Checked && speed > 10000)) lvProxies.Items[proxy.Index].BackColor = Color.FromArgb(255, 207, 206); } else { lvProxies.Items[proxy.Index].SubItems[2].Text = r.Anonymity; lvProxies.Items[proxy.Index].SubItems[3].Text = r.CountryInfo[1]; var countryCode = CountryInfo.GetCode(r.CountryInfo[1]); if (!imageList.Images.Keys.Contains(countryCode)) { imageList.Images.Add(countryCode, Image.FromFile(@"Flags\" + countryCode + ".png")); } lvProxies.Items[proxy.Index].ImageKey = countryCode; lvProxies.Items[proxy.Index].SubItems[4].Text = r.Speed + " ms"; if (cbSSL.Checked) { lvProxies.Items[proxy.Index].SubItems[5].Text = r.SSL ? "Yes" : "No"; lvProxies.Items[proxy.Index].SubItems[5].ForeColor = r.SSL ? Color.DarkGreen : Color.Red; } else { lvProxies.Items[proxy.Index].SubItems[5].Text = "N\\A"; } if (cbCustom.Checked) { lvProxies.Items[proxy.Index].SubItems[6].Text = r.Custom ? "Yes" : "No"; lvProxies.Items[proxy.Index].SubItems[6].ForeColor = r.Custom ? Color.DarkGreen : Color.Red; } else { lvProxies.Items[proxy.Index].SubItems[6].Text = "N\\A"; } lvProxies.Items[proxy.Index].BackColor = Color.FromArgb(202, 255, 202); } })); }); } catch (Exception ex) { MessageBox.Show(ex.ToString()); } }); //lProxyCount.Text = "Loaded Proxies: " + lvProxies.Items.Count; //UpdateStatus("Finished checking proxies."); btnCheck.Enabled = true; proxyChecked = true; //cmsRotator.Enabled = true; //btnRotate.Enabled = true; }
public void Run(string[] args) { if (args.Length == 0) { var dialog = new MainDiscoForm(); dialog.ShowDialog(); return; } bool scanCues = false; string dirArg = null; string infile = null; var loadDiscInterface = DiscInterface.BizHawk; var compareDiscInterfaces = new List <DiscInterface>(); bool hawk = false; int idx = 0; while (idx < args.Length) { string a = args[idx++]; string au = a.ToUpperInvariant(); if (au == "LOAD") { loadDiscInterface = (DiscInterface)Enum.Parse(typeof(DiscInterface), args[idx++], true); } else if (au == "COMPARE") { compareDiscInterfaces.Add((DiscInterface)Enum.Parse(typeof(DiscInterface), args[idx++], true)); } else if (au == "HAWK") { hawk = true; } else if (au == "CUEDIR") { dirArg = args[idx++]; scanCues = true; } else { infile = a; } } if (hawk) { if (infile == null) { return; } // TODO - write it out var dmj = new DiscMountJob { IN_DiscInterface = loadDiscInterface, IN_FromPath = infile }; dmj.Run(); } bool verbose = true; if (scanCues) { verbose = false; var todo = FindCuesRecurse(dirArg); var po = new ParallelOptions(); var cts = new CancellationTokenSource(); po.CancellationToken = cts.Token; po.MaxDegreeOfParallelism = 1; if (po.MaxDegreeOfParallelism < 0) { po.MaxDegreeOfParallelism = 1; } object olock = new object(); int ctr = 0; bool blocked = false; try { Parallel.ForEach(todo, po, (fp) => { lock (olock) { ctr++; int strlen = todo.Count.ToString().Length; string fmt = string.Format("{{0,{0}}}/{{1,{0}}} {{2}}", strlen); Console.WriteLine(fmt, ctr, todo.Count, Path.GetFileNameWithoutExtension(fp)); } if (!blocked) { foreach (var cmpif in compareDiscInterfaces) { var sw = new StringWriter(); bool success = CompareFile(fp, loadDiscInterface, cmpif, verbose, cts, sw); if (!success) { lock (Console.Out) Console.Out.Write(sw.ToString()); cts.Cancel(); return; } } } }); } catch (AggregateException ae) { Console.WriteLine(ae.ToString()); } catch (OperationCanceledException oce) { Console.WriteLine(oce.ToString()); } Console.WriteLine("--TERMINATED--"); return; } if (compareDiscInterfaces.Count != 0) { var sw = new StringWriter(); foreach (var cmpif in compareDiscInterfaces) { CompareFile(infile, loadDiscInterface, cmpif, verbose, null, sw); } sw.Flush(); string results = sw.ToString(); var cr = new ComparisonResults { textBox1 = { Text = results } }; cr.ShowDialog(); } } //Run()
/// <summary> /// 前九字段查询 /// </summary> /// <param name="index"></param> /// <param name="pageSize"></param> /// <returns></returns> public async Task <ResponseModel <t2_house_part_expand> > GetJsonHousePart(int index, int pageSize) { //string sql; //if (index > 1) //{ // sql = $"select top {pageSize} * from t2_house where id > (select max(id) from (select top(({index} - 1) * {pageSize}) id from t2_house order by id) as T) order by id"; //} //else //{ // sql = $"select top {pageSize} * from t2_house"; //} string sql = $"select jsonstr from t2_house order by id offset {pageSize * (index - 1)} row fetch next {pageSize} rows only"; //string sql = $"select top {pageSize} jsonstr from t2_house where id >(select max(id) from (select id from t2_house order by id offset ({pageSize} * ({index} - 1)) row fetch next {pageSize} rows only) as T) order by id"; ConcurrentBag <t2_house_part_expand> t2modelList = new ConcurrentBag <t2_house_part_expand>(); var result = await dataService.GetListAsync <t2_house>(sql); ParallelOptions opt = new ParallelOptions { MaxDegreeOfParallelism = 2 }; //string data = "Data Source=192.168.0.89 ; Initial Catalog = PressureTest; Persist Security Info=True; User ID = sa; Password=qj12345678@"; //ConcurrentBag<t2_house> cb = new ConcurrentBag<t2_house>(); //using (SqlConnection conn = new SqlConnection(data)) //{ // conn.Open(); // string sql = $"select top {pageSize} jsonstr from t2_house where id >(select max(id) from (select id from t2_house order by id offset ({pageSize} * ({index} - 2)) row fetch next {pageSize} rows only) as T) order by id"; // SqlCommand command = new SqlCommand(sql, conn); // DataTable dt = new DataTable(); // SqlDataAdapter da = new SqlDataAdapter(command); // da.Fill(dt); // Parallel.For(0, dt.Rows.Count, i => // { // Parallel.Invoke(() => // { // cb.Add(new t2_house { jsonstr = dt.Rows[i][0].ToString() }); // }); // }); //} Parallel.ForEach(result, opt, item => { var model = JsonConvert.DeserializeObject <t2_house_expand>(item.jsonstr); t2_house_part_expand modeltemp = new t2_house_part_expand(); Parallel.Invoke(() => { T2_ModifyLogModel column4 = model.column4.OrderByDescending(a => a.Column207).FirstOrDefault(); T2_ModifyLogModel column5 = model.column5.OrderByDescending(a => a.Column207).FirstOrDefault(); T2_ModifyLogModel column6 = model.column6.OrderByDescending(a => a.Column207).FirstOrDefault(); T2_ModifyLogModel column7 = model.column7.OrderByDescending(a => a.Column207).FirstOrDefault(); T2_ModifyLogModel column8 = model.column8.OrderByDescending(a => a.Column207).FirstOrDefault(); BaseModel.Mapper(modeltemp, model); modeltemp.column4 = column4.Column205; modeltemp.column5 = column5.Column205; modeltemp.column6 = column6.Column205; modeltemp.column7 = column7.Column205; modeltemp.column8 = column8.Column205; t2modelList.Add(modeltemp); }); }); ResponseModel <t2_house_part_expand> resModel = new ResponseModel <t2_house_part_expand>(t2modelList.ToList().OrderByDescending(o => o.column1).ToList()); return(resModel); }
static void CheckMD5(ProgressReporterDialogue frmProgressReporter, string url) { var baseurl = ConfigurationManager.AppSettings["UpdateLocation"]; if (dobeta) { baseurl = ConfigurationManager.AppSettings["BetaUpdateLocation"]; } L10N.ReplaceMirrorUrl(ref baseurl); string responseFromServer = ""; WebRequest request = WebRequest.Create(url); request.Timeout = 10000; // Set the Method property of the request to POST. request.Method = "GET"; // Get the response. // Get the stream containing content returned by the server. // Open the stream using a StreamReader for easy access. using (WebResponse response = request.GetResponse()) using (Stream dataStream = response.GetResponseStream()) using (StreamReader reader = new StreamReader(dataStream)) { // Display the status. log.Info(((HttpWebResponse)response).StatusDescription); // Read the content. responseFromServer = reader.ReadToEnd(); } Regex regex = new Regex(@"([^\s]+)\s+upgrade/(.*)", RegexOptions.IgnoreCase); if (regex.IsMatch(responseFromServer)) { // background md5 List <Tuple <string, string, Task <bool> > > tasklist = new List <Tuple <string, string, Task <bool> > >(); MatchCollection matchs = regex.Matches(responseFromServer); for (int i = 0; i < matchs.Count; i++) { string hash = matchs[i].Groups[1].Value.ToString(); string file = matchs[i].Groups[2].Value.ToString(); Task <bool> ismatch = Task <bool> .Factory.StartNew(() => MD5File(file, hash)); tasklist.Add(new Tuple <string, string, Task <bool> >(file, hash, ismatch)); } // parallel download ParallelOptions opt = new ParallelOptions() { MaxDegreeOfParallelism = 3 }; Parallel.ForEach(tasklist, opt, task => //foreach (var task in tasklist) { string file = task.Item1; string hash = task.Item2; // check if existing matchs hash task.Item3.Wait(); bool match = task.Item3.Result; if (!match) { log.Info("Newer File " + file); // check is we have already downloaded and matchs hash if (!MD5File(file + ".new", hash)) { if (frmProgressReporter != null) { frmProgressReporter.UpdateProgressAndStatus(-1, Strings.Getting + file); } string subdir = Path.GetDirectoryName(file) + Path.DirectorySeparatorChar; subdir = subdir.Replace("" + Path.DirectorySeparatorChar + Path.DirectorySeparatorChar, "" + Path.DirectorySeparatorChar); GetNewFile(frmProgressReporter, baseurl + subdir.Replace('\\', '/'), subdir, Path.GetFileName(file)); // check the new downloaded file matchs hash if (!MD5File(file + ".new", hash)) { throw new Exception("File downloaded does not match hash: " + file); } } else { log.Info("already got new File " + file); } } else { log.Info("Same File " + file); if (frmProgressReporter != null) { frmProgressReporter.UpdateProgressAndStatus(-1, Strings.Checking + file); } } }); } }
/// <summary> /// One update, ran every like 1/120 seconds /// </summary> /// <param name="e"></param> public void Tick(FrameEventArgs e) { Random r = new Random(); if (currentPoints < numberOfPoints && (running || step)) { for (int i = 0; i < emitters.Length; i++) { if (emitters[i].tickCounter == 0) { for (int j = 0; j < emitters[i].emissionRate; j++) { float Xrad = (float)Math.Sqrt(r.NextDouble()) * emitters[i].Xradius; float Zrad = (float)Math.Sqrt(r.NextDouble()) * emitters[i].Zradius; float angle = (float)r.NextDouble() * (float)Math.PI * 2; Vector3 pos = new Vector3(emitters[i].position.X, emitters[i].position.Y + Xrad * (float)Math.Cos(angle), emitters[i].position.Z + Zrad * (float)Math.Sin(angle)); particles[currentPoints] = (new Sphere(currentPoints, pos, emitters[i].velocity, 0.01f)); particles[currentPoints].color = new Vector3(1.0f, 0, 0); particles[currentPoints].Mass = 1.0f; particles[currentPoints].NetForce = new Vector3(0, 0, 0); currentPoints++; } } emitters[i].tickCounter++; if (emitters[i].tickCounter > emitters[i].delay) { emitters[i].tickCounter = 0; } } } // If set to threading, split the taskforce up, but if the amount of points is too small then there's no point if (threading && !(particles.Length < 100)) { if (running || step) { int PPT = 10; var options = new ParallelOptions() { MaxDegreeOfParallelism = 8 }; Parallel.For(0, currentPoints / PPT, options, i => { simulator.PropertiesUpdate(i * PPT, (i + 1) * PPT); }); Parallel.For(0, currentPoints / PPT, options, i => { simulator.ForcesUpdate(i * PPT, (i + 1) * PPT); }); simulator.MovementUpdate(); for (int i = 0; i < currentPoints; i++) { particles[i].Update(dt); } } } else { //particles[i].Update(dt); if (running || step) { simulator.Update(); step = false; } } }
private void PrefetchingTask_DoWork(object sender, DoWorkEventArgs e) { List <string> illusts = new List <string>(); List <string> originals = new List <string>(); List <string> avatars = new List <string>(); List <string> page_thumbs = new List <string>(); List <string> page_previews = new List <string>(); List <string> needUpdate = new List <string>(); try { var args = e.Argument is PrefetchingOpts ? e.Argument as PrefetchingOpts : new PrefetchingOpts(); if (!args.PrefetchingPreview) { return; } LastStartTime = DateTime.Now; var pagesCount = CalcPagesThumbItems(Items); GetPreviewItems(illusts, avatars, page_thumbs, page_previews, originals); if (pagesCount != page_thumbs.Count + page_previews.Count) { e.Cancel = true; return; } var total = illusts.Count + avatars.Count + page_thumbs.Count + page_previews.Count; if (total <= 0) { e.Cancel = true; return; } var count = total; Percentage = count == 0 ? 100 : 0; Comments = $"Calculating [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; State = TaskStatus.WaitingToRun; if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } needUpdate.AddRange(args.ReverseOrder ? illusts.Reverse <string>() : illusts); needUpdate.AddRange(args.ReverseOrder ? avatars.Reverse <string>() : avatars); needUpdate.AddRange(args.ReverseOrder ? page_thumbs.Reverse <string>() : page_thumbs); needUpdate.AddRange(args.ReverseOrder ? page_previews.Reverse <string>() : page_previews); foreach (var url in needUpdate.Where(url => !PrefetchedList.ContainsKey(url) && File.Exists(url.GetImageCacheFile()))) { PrefetchedList.AddOrUpdate(url, true, (k, v) => true); //if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false); } needUpdate = needUpdate.Where(url => !PrefetchedList.ContainsKey(url) || !PrefetchedList[url]).ToList(); count = needUpdate.Count; Percentage = count == 0 ? 100 : (total - count) / (double)total * 100; if (count == 0) { Comments = $"Done [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; State = TaskStatus.RanToCompletion; } else { Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; State = TaskStatus.Running; } if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } this.DoEvents(); if (count == 0) { return; } var parallels = args.PrefetchingDownloadParallel; if (args.ParallelPrefetching) { var opt = new ParallelOptions(); opt.MaxDegreeOfParallelism = parallels; Parallel.ForEach(needUpdate, opt, (url, loopstate, urlIndex) => { try { var file = url.GetImageCacheFile(); if (!string.IsNullOrEmpty(file)) { if (File.Exists(file)) { PrefetchedList.AddOrUpdate(url, true, (k, v) => true); //if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false); count = count - 1; } else { var _downReport = DownloadProgressActions.ContainsKey(url) ? DownloadProgressActions[url] : null; file = url.DownloadCacheFile(args.Overwrite, progressAction: _downReport, cancelToken: PrefetchingTaskCancelTokenSource).GetAwaiter().GetResult(); if (!string.IsNullOrEmpty(file)) { PrefetchedList.AddOrUpdate(url, true, (k, v) => true); //if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false); count = count - 1; } } } if (PrefetchingBgWorker.CancellationPending) { e.Cancel = true; State = TaskStatus.Canceled; loopstate.Stop(); } Percentage = count == 0 ? 100 : (total - count) / (double)total * 100; Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; State = TaskStatus.Running; if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke((double)this.Percentage, Comments, State); } this.DoEvents(); } catch (Exception ex) { ex.ERROR("PREFETCHING"); } finally { this.DoEvents(); Task.Delay(1).GetAwaiter().GetResult(); } }); } else { SemaphoreSlim tasks = new SemaphoreSlim(parallels, parallels); foreach (var url in needUpdate) { if (PrefetchingBgWorker.CancellationPending) { e.Cancel = true; break; } if (tasks.Wait(-1, PrefetchingTaskCancelTokenSource.Token)) { new Action(async() => { try { var file = url.GetImageCacheFile(); if (!string.IsNullOrEmpty(file)) { if (File.Exists(file)) { PrefetchedList.AddOrUpdate(url, true, (k, v) => true); //if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false); count = count - 1; } else { var _downReport = DownloadProgressActions.ContainsKey(url) ? DownloadProgressActions[url] : null; file = await url.DownloadCacheFile(args.Overwrite, progressAction: _downReport, cancelToken: PrefetchingTaskCancelTokenSource); if (!string.IsNullOrEmpty(file)) { PrefetchedList.AddOrUpdate(url, true, (k, v) => true); //if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false); count = count - 1; } } } if (PrefetchingBgWorker.CancellationPending) { e.Cancel = true; State = TaskStatus.Canceled; return; } Percentage = count == 0 ? 100 : (total - count) / (double)total * 100; Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; State = TaskStatus.Running; if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } //await Task.Delay(10); this.DoEvents(); } catch (Exception ex) { ex.ERROR("PREFETCHING"); } finally { if (tasks is SemaphoreSlim && tasks.CurrentCount <= parallels) { tasks.Release(); } this.DoEvents(); await Task.Delay(1); } }).Invoke(async: false); } } this.DoEvents(); } if (PrefetchingBgWorker.CancellationPending) { e.Cancel = true; State = TaskStatus.Canceled; return; } if (count >= 0 && total > 0) { Percentage = count == 0 ? 100 : (total - count) / (double)total * 100; Comments = $"Done [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]"; //State = TaskStatus.RanToCompletion; //if (ReportProgressSlim is Action) ReportProgressSlim.Invoke(async: false); //else if (ReportProgress is Action<double, string, TaskStatus>) ReportProgress.Invoke(Percentage, Comments, State); //this.DoEvents(); $"Prefetching Previews, Avatars, Thumbnails : {Environment.NewLine} {Comments}".ShowToast("INFO", tag: args.Name ?? Name ?? GetType().Name); } } catch (Exception ex) { ex.ERROR("PREFETCHING"); Comments = $"Failed {Comments}"; State = TaskStatus.Faulted; if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } } finally { try { GetOriginalImageSize(originals, e); if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } this.DoEvents(); illusts.Clear(); avatars.Clear(); page_thumbs.Clear(); page_previews.Clear(); originals.Clear(); needUpdate.Clear(); } catch (Exception ex) { ex.ERROR("PREFETCHED"); } if (CanPrefetching is SemaphoreSlim && CanPrefetching.CurrentCount < 1) { CanPrefetching.Release(); } LastStartTime = DateTime.Now; } }
private bool GetOriginalImageSize(List <string> originals, DoWorkEventArgs e) { var result = false; var setting = Application.Current.LoadSetting(); if (setting.QueryOriginalImageSize) { var args = e.Argument is PrefetchingOpts ? e.Argument as PrefetchingOpts : new PrefetchingOpts(); if (!args.PrefetchingPreview) { return(result); } State = TaskStatus.WaitingToRun; var comments = Comments.Substring(0, Comments.IndexOf("]") + 1); var count = originals.Count; bool paralllel = args.ParallelPrefetching; var parallels = args.PrefetchingDownloadParallel; if (paralllel) { var opt = new ParallelOptions(); opt.MaxDegreeOfParallelism = parallels; Parallel.ForEach(originals, opt, (url, loopstate, urlIndex) => { try { var size = url.QueryImageFileSize(cancelToken: PrefetchingTaskCancelTokenSource).GetAwaiter().GetResult(); if (size > 0) { Comments = comments.Replace("]", $"] [ Q: {--count} / {originals.Count} ]"); if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke((double)Percentage, Comments, State); } this.DoEvents(); } } catch (Exception ex) { ex.ERROR("PREFETCHING"); } finally { this.DoEvents(); Task.Delay(1).GetAwaiter().GetResult(); } }); } else { SemaphoreSlim tasks = new SemaphoreSlim(parallels, parallels); foreach (var url in originals) { if (PrefetchingBgWorker.CancellationPending) { e.Cancel = true; break; } if (tasks.Wait(-1, PrefetchingTaskCancelTokenSource.Token)) { new Action(async() => { try { var size = await url.QueryImageFileSize(cancelToken: PrefetchingTaskCancelTokenSource); if (size > 0) { Comments = comments.Replace("]", $"] [ Q: {--count} / {originals.Count} ]"); if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke((double)Percentage, Comments, State); } this.DoEvents(); } } catch (Exception ex) { ex.ERROR("PREFETCHING"); } finally { if (tasks is SemaphoreSlim && tasks.CurrentCount <= parallels) { tasks.Release(); } this.DoEvents(); await Task.Delay(1); } }).Invoke(async: false); } } } $"Query Original Imagee File Size : {Environment.NewLine} Done [ {originals.Count} ]".ShowToast("INFO", tag: args.Name ?? Name ?? GetType().Name); State = count <= 0 ? TaskStatus.RanToCompletion : TaskStatus.Faulted; if (ReportProgressSlim is Action) { ReportProgressSlim.Invoke(async: false); } else if (ReportProgress is Action <double, string, TaskStatus> ) { ReportProgress.Invoke(Percentage, Comments, State); } setting.SaveImageFileSizeData(); result = true; } return(result); }
private void LoadSpeechRecognition() // fazer o que é preciso para o reconhecimento de voz { sre = new SpeechRecognitionEngine(new System.Globalization.CultureInfo("pt-BR")); // instanciando o reconhecedor passando a cultura da engine sre.SetInputToDefaultAudioDevice(); // definindo o microfone como entrada de aúdio commandsForQA = new List <string>(); commandsForQA.Add("o que é"); commandsForQA.Add("qual é"); commandsForQA.Add("defina"); commandsForQA.Add("definição de"); // vamos processar o AIML aqui Choices cAIML = new Choices(AIML.GetWordsOrSentences()); // obtendo frases e palavras // Vamos ler o arquivo dos comandos string[] cmds = File.ReadAllText("comandos.txt", Encoding.UTF8).Split('$'); // lendo ele e dividindo em linhas Choices cControls = new Choices(); cControls.Add("detecção de movimento"); // Alarme Choices cAlarm = new Choices(); for (int i = 1; i <= 12; i++) { cAlarm.Add(i.ToString()); } // Criação das Gramáticas, Choices Choices cChats = new Choices(); // palavras ou frases de conversa cChats.Add("bom dia"); cChats.Add("boa tarde"); cChats.Add("boa noite"); cChats.Add("jarvis você está ai?"); cChats.Add("ainda acordado jarvis?"); cChats.Add("alguma ideia jarvis?"); cChats.Add("obrigado jarvis"); Choices cDummes = new Choices(); // conversa mais desenrolada cDummes.Add(DummeIn.InStartingConversation.ToArray()); cDummes.Add(DummeIn.InQuestionForDumme.ToArray()); cDummes.Add(DummeIn.InDoWork.ToArray()); cDummes.Add(DummeIn.InDummeStatus.ToArray()); cDummes.Add(DummeIn.InJarvis.ToArray()); Choices cCommands = new Choices(); // palavras ou frases que são comandos // informações de hora e data cCommands.Add("que horas são"); cCommands.Add("que dia é hoje"); cCommands.Add("data de hoje"); cCommands.Add("em que mês estamos"); cCommands.Add("em que ano estamos"); cCommands.Add("minimizar a janela principal"); cCommands.Add("mostrar janela principal"); // Comandos do programa cCommands.Add("exibir lista de comandos"); // status do usuário cCommands.Add("estou com sono"); cCommands.Add("estou indo dormir"); // sair do JARVIS cCommands.Add("até mais jarvis"); // configurar o sintetizador cCommands.Add("pare de falar"); // media player cCommands.Add("media player"); cCommands.Add("selecionar arquivo para o media player"); cCommands.Add("pausar"); cCommands.Add("continuar"); cCommands.Add("parar"); cCommands.Add("fechar media player"); cCommands.Add("abrir diretório para reproduzir"); cCommands.Add("próximo"); cCommands.Add("anterior"); cCommands.Add("aumentar volume do media player"); cCommands.Add("diminuir volume do media player"); cCommands.Add("media player sem som"); cCommands.Add("media player com som"); cCommands.Add("media player em tela cheia"); cCommands.Add("que arquivo está tocando"); // informações do sistema cCommands.Add("em quanto estar o uso do processador?"); cCommands.Add("quanta memória ram estar sendo usada?"); cCommands.Add("quanta mamória ram ainda há livre?"); cCommands.Add("quanta memória ram há no total?"); // Comandos, adicionar cCommands.Add("adicionar novo comando"); // processos cCommands.Add("detalhes dos processos"); // processList cCommands.Add("lista de processos"); cCommands.Add("fechar o processo selecionado"); // jarvis cCommands.Add("introdução ao assistente jarvis"); cCommands.Add("desligar computador"); cCommands.Add("reiniciar computador"); cCommands.Add("cancelar desligamento"); cCommands.Add("cancelar reinicialização"); // Youtube cCommands.Add("tocar algo do youtube"); cCommands.Add("adicionar link para o youube"); cCommands.Add("previsão do tempo"); // controle de janelas cCommands.Add("alterar de janela"); cCommands.Add("fechar janela"); // comandos de teclas cCommands.Add("copiar texto selecionado"); cCommands.Add("colar texto selecionado"); cCommands.Add("salvar este arquivo"); cCommands.Add("selecionar tudo"); cCommands.Add("nova linha"); // Comandos do canal Código Logo cCommands.Add("reproduza um vídeo do canal"); //Choices cNumbers = new Choices(File.ReadAllLines("n.txt")); // números Choices cProcess = new Choices(); // lista de comandos cProcess.Add("bloco de notas"); cProcess.Add("windows media player"); cProcess.Add("prompt de comando"); cProcess.Add("gerenciador de tarefas"); cProcess.Add("minhas pastas"); cProcess.Add("calculadora"); cProcess.Add("mapa de caracteres"); cProcess.Add("limpeza de disco"); cProcess.Add("gerenciamento de cores"); cProcess.Add("serviços de componente"); cProcess.Add("gerenciamento de computador"); cProcess.Add("definir programas padrão"); cProcess.Add("painel de controle"); cProcess.Add("otimizador de texto"); cProcess.Add("calibragem de cores"); cProcess.Add("desfragmentador de disco"); cProcess.Add("adicionar um novo dispositivo"); cProcess.Add("gerenciador de dispositivos"); cProcess.Add("discagem telefônica"); cProcess.Add("gerenciamento de disco"); Choices cCustomSites = new Choices(); // lista de comandos do usuário Choices webSearch = new Choices(); StreamReader srWords = new StreamReader("words.txt", Encoding.UTF8); while (srWords.Peek() >= 0) { try { webSearch.Add(srWords.ReadLine()); } catch { } } srWords.Close(); // vamos processar o comandos.txt for (int i = 0; i < cmds.Length; i++) { try { if (cmds[i].StartsWith("site#")) { cmds[i] = cmds[i].Replace("site#", ""); string[] temp = cmds[i].Split('#'); cCustomSites.Add(temp[0]); // adicionamos a palavra na gramática dictCmdSites.Add(temp[0], temp[1]); } } catch { } } // Gramática do alarme GrammarBuilder gbAlarm = new GrammarBuilder(); gbAlarm.Append(new Choices("defina alarme", "alarme às", "despertador às")); gbAlarm.Append(cAlarm); gbAlarm.Append(new Choices("horas da manhã", "horas da tarde", "horas da noite")); // GrammarsBuilders GrammarBuilder gbChats = new GrammarBuilder(); // vamos criar um grammaBuilder para as conversas gbChats.Append(cChats); // já foi feito GrammarBuilder gbDumme = new GrammarBuilder(); // conversa solta gbDumme.Append(cDummes); GrammarBuilder gbCommands = new GrammarBuilder(); //para a lista de comandos gbCommands.Append(cCommands); // feito GrammarBuilder gbControls = new GrammarBuilder(); gbControls.Append(cControls); /* * GrammarBuilder gbCalculations = new GrammarBuilder(); // gramática que vai fazer cálculos * gbCalculations.Append("quanto é"); // primeira parte * gbCalculations.Append(cNumbers); // números * gbCalculations.Append(new Choices("mais", "menos", "dividido por", "vezes", "porcento de")); * gbCalculations.Append(cNumbers); // números novamente * // essa gramática pode fazer por exemplo "quanto é 65 vezes 42", "quanto é 14 porcento de 500" */ GrammarBuilder gbProcess = new GrammarBuilder(); gbProcess.Append(new Choices("abrir", "fechar")); // comando gbProcess.Append(cProcess); // adicionar lista de processos GrammarBuilder gbCustomSites = new GrammarBuilder(); // sites e páginas gbCustomSites.Append(new Choices("abrir", "iniciar", "carregar", "ir para o")); // parametros gbCustomSites.Append(cCustomSites); GrammarBuilder gbWebSearch = new GrammarBuilder(); gbWebSearch.Append(new Choices("pesquisar", "buscar", "procurar", "buscar por", "pesquisar vídeo de", "imagem de", "imagens de", "pesquisar imagem de", "pesquisar por")); gbWebSearch.Append(webSearch); gbWebSearch.Append(new Choices("no youtube", "no google")); GrammarBuilder gbQA = new GrammarBuilder(); // gramática para responder perguntas gbQA.Append(new Choices(commandsForQA.ToArray())); gbQA.Append(cAIML); // Grammars Grammar gQA = new Grammar(gbQA); gQA.Name = "QA"; Grammar gChats = new Grammar(gbChats); // gramática das conversas gChats.Name = "Chats"; // damos um nome para a gramática, pois vamos usa isso mais adiante Grammar gDumme = new Grammar(gbDumme); gDumme.Name = "Dumme"; // nome Grammar gCommands = new Grammar(gbCommands); // gramática dos comandos gCommands.Name = "Commands"; // nome Grammar gCustomSites = new Grammar(gbCustomSites); // gramáticas dos sites gCustomSites.Name = "Sites"; Grammar gAIML = new Grammar(new GrammarBuilder(cAIML)); gAIML.Name = "AIML"; /* * Grammar gCalculations = new Grammar(gbCalculations); * gCalculations.Name = "Calculations"; */ Grammar gProcess = new Grammar(gbProcess); gProcess.Name = "Process"; // Agora vamos carregar as gramáticas Grammar gWebSearch = new Grammar(gbWebSearch); gWebSearch.Name = "Search"; Grammar gControls = new Grammar(gbControls); gControls.Name = "Control"; // podemos fazer de várias maneiras, por enquanto vou fazer o seguinte // Lista de gramáticas List <Grammar> grammars = new List <Grammar>(); grammars.Add(gQA); grammars.Add(gChats); grammars.Add(gDumme); grammars.Add(gCommands); // comandos grammars.Add(gAIML); grammars.Add(gWebSearch); grammars.Add(gControls); //grammars.Add(gCalculations); grammars.Add(gProcess); grammars.Add(gCustomSites); ParallelOptions op = new ParallelOptions() { MaxDegreeOfParallelism = 4 }; Parallel.For(0, grammars.Count, op, i => // loop paralelo { sre.LoadGrammar(grammars[i]); // carregar gramática }); speechRecognitionActived = true; // reconhecimento de voz ativo! sre.SpeechRecognized += new EventHandler <SpeechRecognizedEventArgs>(reconhecido); // evento do reconhecimento sre.AudioLevelUpdated += new EventHandler <AudioLevelUpdatedEventArgs>(audioElevou); // quando o aúdio é elevadosre sre.SpeechRecognitionRejected += new EventHandler <SpeechRecognitionRejectedEventArgs>(rejeitado); // quando o reconhecimento de voz falhou sre.SpeechDetected += new EventHandler <SpeechDetectedEventArgs>(vozDetectada); // alguma voz foi detectada sre.LoadGrammarCompleted += new EventHandler <LoadGrammarCompletedEventArgs>(loaded); // gramática carregada sre.RecognizeAsync(RecognizeMode.Multiple); // iniciar o reconhecimento async e múltiplo }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int folderThreadCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null) { var allDirectories = new Stack <string>(); var allFailedFiles = new ConcurrentDictionary <string, string>(); var allFailedDirs = new List <string>(); var fileCount = 0; var testFileCountChanged = 0; var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive); var folderPathStartIndex = Path.GetDirectoryName(sourceFolderPath).Length; if (folderPathStartIndex < 1) { // this is the scenario where the user is copying from the root of a drive // such as C:\ or .\. In these cases, we simply indicate the "beginning" as the // end of the root. folderPathStartIndex = sourceFolderPath.Length; } allDirectories.Push(sourceFolderPath); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes to be copied: {2}. Total files to be copied: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount; var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; //TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in while (allDirectories.Count > 0) { var currentDir = allDirectories.Pop(); string[] files; try { files = Directory.GetFiles(currentDir); if (recursive) { // Push the subdirectories onto the stack for traversal. // This could also be done before handing the files. foreach (var str in Directory.GetDirectories(currentDir)) { allDirectories.Push(str); } } } catch { // update the list of folders that could not be accessed // for later reporting to the user. allFailedDirs.Add(currentDir); continue; } // Execute in parallel if there are enough files in the directory. // Otherwise, execute sequentially.Files are opened and processed // synchronously but this could be modified to perform async I/O. // NOTE: in order to write progress in a meaningful way, we have // wrapped the parallel execution in a container task, which is // then monitored from the main thread. // TODO: enable resumability in the event that copy fails somewhere in the middle var folderOptions = new ParallelOptions { CancellationToken = cmdletCancellationToken }; if (folderThreadCount > 0) { folderOptions.MaxDegreeOfParallelism = folderThreadCount; } var task = Task.Run( () => { Parallel.ForEach( files, folderOptions, () => 0, (file, loopState, localCount) => { cmdletCancellationToken.ThrowIfCancellationRequested(); var dataLakeFilePath = string.Format( "{0}/{1}", destinationFolderPath, file.Substring(folderPathStartIndex).TrimStart('\\').Replace('\\', '/')); // for each file we will either honor a force conversion // to either binary or text, or attempt to determine // if the file is either binary or text, with a default // behavior of text. isBinary = forceBinaryOrText ? isBinary : GlobalMembers.BinaryFileExtension.Contains( Path.GetExtension(file).ToLowerInvariant()); try { CopyFile(dataLakeFilePath, accountName, file, cmdletCancellationToken, internalFileThreads, overwrite, resume, isBinary, null, progress); } catch (Exception e) { allFailedFiles.GetOrAdd(file, e.Message); } // note: we will always increment the count, since the file was seen and attempted // this does not necessarily mean the file was successfully uploaded, as indicated by // the warning messages that can be written out. return(++localCount); }, c => Interlocked.Add(ref fileCount, c)); }, cmdletCancellationToken); while (!task.IsCompleted && !task.IsCanceled) { // if we somehow made it in here prior to the cancel, I want to issue a throw cmdletCancellationToken.ThrowIfCancellationRequested(); // only update progress if the percentage has changed. if ((int)Math.Ceiling((decimal)testFileCountChanged / totalFiles * 100) < (int)Math.Ceiling((decimal)fileCount / totalFiles * 100)) { testFileCountChanged = fileCount; var percentComplete = (int)Math.Ceiling((decimal)fileCount / totalFiles * 100); if (percentComplete > 100) { // in some cases we can get 101 percent complete using ceiling, however we want to be // able to round up to full percentage values, instead of down. percentComplete = 100; } progress.PercentComplete = percentComplete; UpdateProgress(progress, cmdletRunningRequest); } // sleep for a half of a second. TestMockSupport.Delay(500); } if (task.IsFaulted && !task.IsCanceled) { var ae = task.Exception; if (ae != null) { if (cmdletRunningRequest != null) { cmdletRunningRequest.WriteWarning( "The following errors were encountered during the copy:"); } else { Console.WriteLine(@"The following errors were encountered during the copy:"); } ae.Handle( ex => { if (ex is AggregateException) { var secondLevel = ex as AggregateException; secondLevel.Handle( secondEx => { if (cmdletRunningRequest != null) { cmdletRunningRequest.WriteWarning(secondEx.ToString()); } else { Console.WriteLine(secondEx); } return(true); }); } else { if (cmdletRunningRequest != null) { cmdletRunningRequest.WriteWarning(ex.ToString()); } else { Console.WriteLine(ex); } } return(true); }); } } } if (allFailedDirs.Count > 0 && !cmdletCancellationToken.IsCancellationRequested) { var errString = "The following {0} directories could not be opened and their contents must be copied up with the single file copy command: {1}"; if (cmdletRunningRequest != null) { cmdletRunningRequest.WriteWarning( string.Format(errString, allFailedDirs.Count, string.Join(",\r\n", allFailedDirs))); } else { Console.WriteLine(errString, allFailedDirs.Count, string.Join(",\r\n", allFailedDirs)); } } if (allFailedFiles.Count > 0 && !cmdletCancellationToken.IsCancellationRequested) { var errString = "The following {0} files could not be copied and must be copied up with the single file copy command: {1}"; if (cmdletRunningRequest != null) { cmdletRunningRequest.WriteWarning( string.Format(errString, allFailedFiles.Count, string.Join(",\r\n", allFailedFiles))); } else { Console.WriteLine(errString, allFailedFiles.Count, string.Join(",\r\n", allFailedFiles)); } } if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
/// <summary>Renders a mandelbrot fractal.</summary> /// <param name="position">The MandelbrotPosition representing the fractal boundaries to be rendered.</param> /// <param name="imageWidth">The width in pixels of the image to create.</param> /// <param name="imageHeight">The height in pixels of the image to create.</param> /// <param name="parallelRendering">Whether to render the image in parallel.</param> /// <returns>The rendered Bitmap.</returns> public unsafe static Bitmap Create(MandelbrotPosition position, int imageWidth, int imageHeight, CancellationToken cancellationToken, bool parallelRendering) { // The maximum number of iterations to perform for each pixel. Higher number means better // quality but also slower. const int maxIterations = 256; // In order to use the Bitmap ctor that accepts a stride, the stride must be divisible by four. // We're using imageWidth as the stride, so shift it to be divisible by 4 as necessary. if (imageWidth % 4 != 0) { imageWidth = (imageWidth / 4) * 4; } // Based on the fractal bounds, determine its upper left coordinate double left = position.CenterX - (position.Width / 2); double top = position.CenterY - (position.Height / 2); // Get the factors that can be multiplied by row and col to arrive at specific x and y values double colToXTranslation = position.Width / (double)imageWidth; double rowToYTranslation = position.Height / (double)imageHeight; // Create the byte array that will store the rendered color indices int pixels = imageWidth * imageHeight; byte[] data = new byte[pixels]; // initialized to all 0s, which equates to all black based on the default palette // Generate the fractal using the mandelbrot formula : z = z^2 + c // Parallel implementation if (parallelRendering) { var options = new ParallelOptions { CancellationToken = cancellationToken }; Parallel.For(0, imageHeight, options, row => { double initialY = row * rowToYTranslation + top; fixed(byte *ptr = data) { byte *currentPixel = &ptr[row * imageWidth]; for (int col = 0; col < imageWidth; col++, currentPixel++) { Complex c = new Complex(col * colToXTranslation + left, initialY); Complex z = c; for (int iteration = 0; iteration < maxIterations; iteration++) { if (z.Magnitude > 4) { *currentPixel = (byte)iteration; break; } z = (z * z) + c; } } } }); } // Sequential implementation else { for (int row = 0; row < imageHeight; row++) { cancellationToken.ThrowIfCancellationRequested(); double initialY = row * rowToYTranslation + top; fixed(byte *ptr = data) { byte *currentPixel = &ptr[row * imageWidth]; for (int col = 0; col < imageWidth; col++, currentPixel++) { Complex c = new Complex(col * colToXTranslation + left, initialY); Complex z = c; for (int iteration = 0; iteration < maxIterations; iteration++) { if (z.Magnitude > 4) { *currentPixel = (byte)iteration; break; } z = (z * z) + c; } } } } ; } // Produce a Bitmap from the byte array of color indices and return it fixed(byte *ptr = data) { using (Bitmap tempBitmap = new Bitmap(imageWidth, imageHeight, imageWidth, PixelFormat.Format8bppIndexed, (IntPtr)ptr)) { Bitmap bitmap = tempBitmap.Clone(new Rectangle(0, 0, tempBitmap.Width, tempBitmap.Height), PixelFormat.Format8bppIndexed); UpdatePalette(bitmap); return(bitmap); } } }
public static void WriteTypes(Dictionary <string, ParsedType> types, string outputDirectory) { var di = new System.IO.DirectoryInfo(outputDirectory); string apibase = di.Name; var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 100 }; Parallel.ForEach(types, parallelOptions, (keyValue) => { ParsedType basetype = keyValue.Value; if (!basetype.IsPublic) { return; } var content = new StringBuilder(); content.AppendLine("---"); content.AppendLine($"title: \"{basetype.Name}\""); content.AppendLine($"date: {DateTime.Now.ToString("u")}"); content.AppendLine("draft: false"); content.AppendLine("---"); content.AppendLine(); content.AppendLine($"*Namespace: [{basetype.Namespace}](../)*"); content.AppendLine(); string baseTypeSummary = basetype.Summary(); if (!string.IsNullOrEmpty(baseTypeSummary)) { content.AppendLine(baseTypeSummary); } if (basetype.IsClass) { content.AppendLine("```cs"); string[] attributes = basetype.GetAttributes(); for (int i = 0; i < attributes.Length; i++) { content.AppendLine(attributes[i]); } content.Append($"public class {basetype.Name}"); string[] baseList = basetype.GetBaseList(null); if (baseList != null) { for (int i = 0; i < baseList.Length; i++) { if (i == 0) { content.Append($": {baseList[i]}"); } else { content.Append($", {baseList[i]}"); } } } content.AppendLine(); content.AppendLine("```"); } if (basetype.Members == null) { return; // TODO: Figure out this case } ParsedMemberType state = ParsedMemberType.None; foreach (var item in basetype.Members) { if (item.IsEvent && state != ParsedMemberType.Event) { content.AppendLine("## Events"); state = ParsedMemberType.Event; } if (item.IsProperty && state != ParsedMemberType.Property) { content.AppendLine("## Properties"); state = ParsedMemberType.Property; } if (item.IsMethod && state != ParsedMemberType.Method) { content.AppendLine("## Methods"); state = ParsedMemberType.Method; } if (item.IsConstructor && state != ParsedMemberType.Constructor) { content.AppendLine("## Constructors"); state = ParsedMemberType.Constructor; } content.AppendLine(); string signature = item.Signature(false); var returntype = item.ReturnType(types.Values); if (returntype != null && returntype != item.ParentType) { string rn = returntype.Name; int index = signature.IndexOf(rn); string link = ParsedTypePath(apibase, returntype); string s = ""; if (index > 0) { s = signature.Substring(0, index); } s += $"[{signature.Substring(index, rn.Length)}]({link}){signature.Substring(index + rn.Length)}"; signature = s; } content.AppendLine(signature); content.AppendLine($": {item.Summary()}"); string returnString = item.ReturnDocString(); if (!string.IsNullOrWhiteSpace(returnString)) { content.AppendLine($": Returns - {returnString}"); } if (!item.Since.Equals("5.0")) // no need to add since tags initial items { content.AppendLine($": since {item.Since}"); } } string name = basetype.Name; string directory = OutputDirectoryFromNamespace(outputDirectory, basetype.Namespace); string path = System.IO.Path.Combine(directory, name.ToLower() + ".md"); if (WriteContent(content, path, true)) { Console.WriteLine($"(write) {name}"); } else { Console.WriteLine($"(no change) {name}"); } }); }