public Mcsrch(int thread_num)
{
infoc = 0;
stage1 = false;
brackt = false;
finit = 0.0;
dginit = 0.0;
dgtest = 0.0;
width = 0.0;
width1 = 0.0;
stx = 0.0;
fx = 0.0;
dgx = 0.0;
sty = 0.0;
fy = 0.0;
dgy = 0.0;
stmin = 0.0;
stmax = 0.0;
#if NO_SUPPORT_PARALLEL_LIB
#else
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
}
/// <summary>
/// Запускает на выполнение задачу по скачиванию песен и возвращает результат
/// </summary>
/// <param name="CancToken">Токен отмены операции</param>
/// <param name="MaxDegreeOfParallelism">Максимальное количество потоков, которое будет использоваться для запросов к серверу.
/// Если меньше 1, ограничение на количество потоков будет снято.</param>
/// <returns></returns>
public IDictionary<OneSongHeader, Exception> Start(CancellationToken CancToken, Int32 MaxDegreeOfParallelism)
{
if (MaxDegreeOfParallelism < 1) { MaxDegreeOfParallelism = -1; }
ConcurrentDictionary<OneSongHeader, Exception> intermediate =
new ConcurrentDictionary<OneSongHeader, Exception>(MaxDegreeOfParallelism, this._songs.Length);
ParallelOptions opt = new ParallelOptions() { CancellationToken = CancToken, MaxDegreeOfParallelism = MaxDegreeOfParallelism };
try
{
ParallelLoopResult p_res = Parallel.ForEach(this._songs, opt,
(OneSongHeader song, ParallelLoopState pls, Int64 i) =>
{
if (pls.ShouldExitCurrentIteration)
{
pls.Stop();
}
KeyValuePair<OneSongHeader, Exception> res =
Core.DownloadAndSaveOneSong
(song, this._userAgent, this._generateNewFilenames, this._filenameTemplate, this._folderPath, (Int32)i + 1);
this.OnNext.Invoke(res.Key, res.Value);
intermediate.TryAdd(res.Key, res.Value);
}
);
}
catch (OperationCanceledException)
{
this.OnCancellation(intermediate.Count, this._songs.Length);
CancToken.ThrowIfCancellationRequested();
}
this.OnComplete.Invoke(intermediate);
return intermediate;
}
public object Solve()
{
int answer = 0;
CancellationTokenSource tokenSource = new CancellationTokenSource();
var options = new ParallelOptions() { CancellationToken = tokenSource.Token, MaxDegreeOfParallelism = 4 };
try
{
Parallel.ForEach(TriangleNumbers().Skip(500), options, num =>
{
var count = Divisors(num).Count();
if (count > 500)
{
tokenSource.Cancel();
answer = num;
}
});
}
catch (OperationCanceledException)
{
// no op
}
return answer;
}
public virtual async Task<CodeAction> GetFixAsync(
ImmutableDictionary<Document, ImmutableArray<Diagnostic>> documentsAndDiagnosticsToFixMap,
FixAllContext fixAllContext)
{
if (documentsAndDiagnosticsToFixMap != null && documentsAndDiagnosticsToFixMap.Any())
{
fixAllContext.CancellationToken.ThrowIfCancellationRequested();
var documents = documentsAndDiagnosticsToFixMap.Keys.ToImmutableArray();
var fixesBag = new List<CodeAction>[documents.Length];
var options = new ParallelOptions() { CancellationToken = fixAllContext.CancellationToken };
Parallel.ForEach(documents, options, (document, state, index) =>
{
fixAllContext.CancellationToken.ThrowIfCancellationRequested();
fixesBag[index] = new List<CodeAction>();
this.AddDocumentFixesAsync(document, documentsAndDiagnosticsToFixMap[document], fixesBag[index].Add, fixAllContext).Wait(fixAllContext.CancellationToken);
});
if (fixesBag.Any(fixes => fixes.Count > 0))
{
return await this.TryGetMergedFixAsync(fixesBag.SelectMany(i => i), fixAllContext).ConfigureAwait(false);
}
}
return null;
}
static void Main(string[] args)
{
int start = Environment.TickCount;
int maxParallelUploads = 50;
string urlForUploads = "http://localhost:8080";
string dirToUpload = Path.Combine(".", "default-to-upload");
if (args.Length > 0)
{
dirToUpload = args[0];
}
ParallelOptions opts = new ParallelOptions();
opts.MaxDegreeOfParallelism = maxParallelUploads;
Parallel.ForEach(Directory.EnumerateFiles(dirToUpload, "*", SearchOption.AllDirectories), opts, (filepath) =>
{
Console.WriteLine("Uploading {0} on thread {1}...", filepath, Thread.CurrentThread.ManagedThreadId);
WebClient webClient = new WebClient();
int sleepPeriodMs = 1000;
bool retry = true;
bool success = false;
while (retry)
{
retry = false;
try
{
webClient.UploadFile(urlForUploads, filepath);
success = true;
}
catch (WebException e)
{
var r = (HttpWebResponse)e.Response;
if (r != null && r.StatusCode == HttpStatusCode.ServiceUnavailable)
{
// We are overloading the server. Wait some time and try again.
Console.WriteLine("Server is overloaded. Retrying in {0}ms...", sleepPeriodMs);
Thread.Sleep(sleepPeriodMs);
sleepPeriodMs *= 2;
retry = true;
}
else
{
Console.WriteLine("Failed to upload file {0}. Error was: \n{1}.\nMoving on to next file.", filepath, e.ToString());
}
}
catch
{
Console.WriteLine("Unexpected error! Failed to upload file {0}. Moving on to next file.", filepath);
}
}
if (success)
{
// The file was successfully uploaded to the server - delete it!
File.Delete(filepath);
}
});
Console.WriteLine("Took {0} ticks to upload files.", Environment.TickCount - start);
}
public Form1()
{
InitializeComponent();
cancellationTokenSource = new CancellationTokenSource();
options = new ParallelOptions();
options.CancellationToken = cancellationTokenSource.Token;
}
private void Execute(IStatSource state, ParallelOptions parallelOptions)
{
state.Calculate(parallelOptions.CancellationToken);
lock (_statLock) {
_pendingStatJobs--;
}
}
private void DoWork(object unused)
{
try
{
var fileList = GetFiles();
var parallelOptions = new ParallelOptions();
parallelOptions.CancellationToken = _cancellationToken;
parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
Parallel.For(0, fileList.Count, parallelOptions, i =>
{
string file = fileList[i];
IEnumerable<ParseItem> items = GetItems(file, _searchValue);
foreach (ParseItem sel in items)
{
_cancellationToken.ThrowIfCancellationRequested();
_navigateToCallback.AddItem(new NavigateToItem(_searchValue, NavigateToItemKind.Field, "CSS", _searchValue, new CssGoToLineTag(sel, file), MatchKind.Exact, _providerFactory));
}
var backgroundProgress = Interlocked.Increment(ref _backgroundProgress);
_navigateToCallback.ReportProgress(backgroundProgress, fileList.Count);
});
}
catch
{
// Don't let exceptions from the background thread reach the ThreadPool. Swallow them
// here and complete the navigate operation
}
finally
{
_navigateToCallback.Done();
}
}
private static void CancellationOfParallelFor()
{
var nums = Enumerable.Range(0, 100000);
var cts = new CancellationTokenSource();
var options = new ParallelOptions()
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Console.WriteLine("Press any key to start, Press 'c' to cancel");
Console.ReadKey();
Task.Factory.StartNew(() =>
{
if (Console.ReadKey().KeyChar == 'c')
cts.Cancel();
});
try
{
Parallel.ForEach(nums, options, num =>
{
var d = Math.Sqrt(num);
Console.WriteLine("{0} on {1}", d,
Thread.CurrentThread.ManagedThreadId);
options.CancellationToken.ThrowIfCancellationRequested();
});
}
catch (OperationCanceledException ex)
{
Console.WriteLine(ex.Message);
}
}
public void DroppingConnectionsTest()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 1000
};
var clusterInfo = TestUtils.CcmSetup(1);
var session = clusterInfo.Session;
//For a node to be back up could take up to 60 seconds
const int bringUpNodeMilliseconds = 60000;
try
{
Action dropConnections = () =>
{
session.Execute("SELECT * FROM system.schema_keyspaces");
TestUtils.CcmStopForce(clusterInfo, 1);
Thread.Sleep(2000);
TestUtils.CcmStart(clusterInfo, 1);
};
Action query = () =>
{
Thread.Sleep(bringUpNodeMilliseconds);
//All the nodes should be up but the socket connections are not valid
session.Execute("SELECT * FROM system.schema_keyspaces");
};
Parallel.Invoke(parallelOptions, dropConnections, query);
}
finally
{
TestUtils.CcmRemove(clusterInfo);
}
}
static void Foo1()
{
var list = new List<int>() {10,20,30,40,50 };
var options = new ParallelOptions();
var total = 0;
var result = Parallel.For(0, list.Count,
() =>
{
Console.WriteLine("Thread");
return 1;
},
(i, loop, j) =>
{
Console.WriteLine("body");
Console.WriteLine("i= " + list[i] + " j=" + j);
return list[i];
},
(i) =>
{
Console.WriteLine("Foot");
Interlocked.Add(ref total,i);
Console.WriteLine("total="+total);
});
Console.WriteLine(result.IsCompleted);
}
public virtual async Task<CodeAction> GetFixAsync(
ImmutableDictionary<Document, ImmutableArray<Diagnostic>> documentsAndDiagnosticsToFixMap,
FixAllContext fixAllContext)
{
if (documentsAndDiagnosticsToFixMap != null && documentsAndDiagnosticsToFixMap.Any())
{
FixAllLogger.LogDiagnosticsStats(documentsAndDiagnosticsToFixMap);
var fixesBag = new ConcurrentBag<CodeAction>();
using (Logger.LogBlock(FunctionId.CodeFixes_FixAllOccurrencesComputation_Fixes, fixAllContext.CancellationToken))
{
fixAllContext.CancellationToken.ThrowIfCancellationRequested();
var documents = documentsAndDiagnosticsToFixMap.Keys.ToImmutableArray();
var options = new ParallelOptions() { CancellationToken = fixAllContext.CancellationToken };
Parallel.ForEach(documents, options, document =>
{
fixAllContext.CancellationToken.ThrowIfCancellationRequested();
AddDocumentFixesAsync(document, documentsAndDiagnosticsToFixMap[document], fixesBag.Add, fixAllContext).Wait(fixAllContext.CancellationToken);
});
}
if (fixesBag.Any())
{
using (Logger.LogBlock(FunctionId.CodeFixes_FixAllOccurrencesComputation_Merge, fixAllContext.CancellationToken))
{
FixAllLogger.LogFixesToMergeStats(fixesBag);
return await TryGetMergedFixAsync(fixesBag, fixAllContext).ConfigureAwait(false);
}
}
}
return null;
}
public void SearchBattle()
{
var opt = new ParallelOptions();
opt.MaxDegreeOfParallelism = 4;
Parallel.ForEach(Players.Where(p => !p.isBattle).OrderBy(p=>GB.Random.NextDouble()).ToList(), opt, () => new List<Battle>(), (player, pls, battles) =>
{
if (Battles.Find(p => p != null && p.CanJoin(player.Udemae) && p.Join(player)) == null)
{
if (battles.Find(p => p != null && p.CanJoin(player.Udemae) && p.Join(player)) == null)
{
var battle = new Battle(player.Udemae);
lock (this)
{
battles.Add(battle);
battle.Join(player);
}
}
}
return battles;
}
,
(battles) =>
{
lock (this)
{
Battles.AddRange(battles);
}
}
);
}
protected override void Run(CancellationToken cancellationToken) {
this.cancellationToken = cancellationToken;
parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = DegreeOfParallelism;
parallelOptions.CancellationToken = cancellationToken;
Run(ExecutionStack);
}
public void ExternalBreak(CancellationTokenSource cancellationTokenSource)
{
var cancellationToken = cancellationTokenSource.Token;
var options = new ParallelOptions { CancellationToken = cancellationToken };
try
{
Parallel.For(
0,
100,
options,
i =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
Console.WriteLine(i);
Thread.Sleep(500);
});
}
catch (OperationCanceledException ex)
{
Console.WriteLine(ex.Message);
}
}
public static void Main ()
{
object count_lock = new object ();
int count = 0;
ParallelOptions options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount * 4,
};
Thread t1 = new Thread (() => {
Parallel.ForEach (UntilTimeout (15 * 1000), options, _ => {
using (Process p = Process.Start ("cat", "/dev/null")) {
p.WaitForExit ();
}
lock (count_lock) {
count += 1;
if (count % (10) == 0)
Console.Write (".");
if (count % (10 * 50) == 0)
Console.WriteLine ();
}
});
});
t1.Start ();
while (!t1.Join (0)) {
try {
using (Process p = Process.GetProcessById (1));
} catch (ArgumentException) {
}
}
}
static void Main()
{
var tokenSource = new CancellationTokenSource();
var options = new ParallelOptions { CancellationToken = tokenSource.Token };
ThreadPool.QueueUserWorkItem(obj =>
{
try
{
Parallel.For(0, 1000, options, i => DoWork(i, options.CancellationToken));
}
catch (OperationCanceledException)
{
Console.WriteLine("Loop operation was cancelled");
}
});
Thread.Sleep(250);
Console.WriteLine("Cancelling work...");
tokenSource.Cancel();
Thread.Sleep(1000);
}
async Task<IDictionary<string, CatalogItemSummary>> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken)
{
ConcurrentDictionary<string, CatalogItemSummary> pageItems = new ConcurrentDictionary<string, CatalogItemSummary>();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = _threads;
var items = _batch.ToArray();
ConcurrentDictionary<CatalogItem, Uri> tmpPages = new ConcurrentDictionary<CatalogItem, Uri>();
Parallel.ForEach(items, options, item =>
{
Uri resourceUri = null;
try
{
item.TimeStamp = commitTimeStamp;
item.CommitId = commitId;
item.BaseAddress = Storage.BaseAddress;
Uri catalogPageUri = CreateCatalogPage(item);
//CommitItemComplete(catalogPageUri);
resourceUri = item.GetItemAddress();
if (!tmpPages.TryAdd(item, catalogPageUri))
{
throw new Exception("duplicate item");
}
//if (catalogPageUri != null)
//{
// Uri indexPageUri = CreateIndexEntry(item, catalogPageUri, commitId, commitTimeStamp);
// CommitItemComplete(catalogPageUri, indexPageUri);
//}
//else
//{
// Debug.Fail("Missing catalog content");
//}
}
catch (Exception e)
{
throw new Exception(string.Format("item uri: {0}", resourceUri == null ? "none" : resourceUri.AbsoluteUri), e);
}
});
// make sure to commit these in the correct order
foreach (var item in items)
{
Uri pageUri = null;
tmpPages.TryGetValue(item, out pageUri);
CommitItemComplete(pageUri);
}
return pageItems;
}
public static Info search(String path, String extension, string predicate)
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ParallelOptions options = new ParallelOptions();
options.CancellationToken = cancellationTokenSource.Token;
Info info = new Info();
string[] files = Directory.GetFiles(path, extension, SearchOption.AllDirectories);
info.totalextensions = files.Length;
ParallelLoopResult loopResult = Parallel.For(
0,
files.Length,
options,
(i, loopState) =>
{
if (File.ReadAllText(files[i]).Contains(predicate))
{
info.names.Add(files[i]);
}
}
);
if (loopResult.IsCompleted)
{
info.total = info.names.Count;
}
return info;
}
public static ParallelLoopResult For( int start, int end, ParallelOptions options, Action<int> body )
{
// Following the HAPPY PATH :)
// Problem : If the division result is a decimal number
var segment = ( ( end - start ) / options.MaxDegreeOfParallelism );
var totalSplits = ( end - start ) / segment;
Task task = new Task( () =>
{
for ( int i = 0 ; i < totalSplits ; ++i )
{
int i2 = i;
Task.Factory.StartNew( () =>
{
int i1 = i2;
Console.WriteLine( "Start -> {0}", start + ( i1 * segment ) );
Console.WriteLine( "End -> {0}", start + segment + ( i1 * segment ) );
for ( int j = start + ( i1 * segment ) ; j < start + segment + ( i1 * segment ) ; ++j )
{
body( j );
}
},
TaskCreationOptions.AttachedToParent );
}
}
);
task.Start( options.TaskScheduler );
task.Wait( options.CancellationToken );
return Parallel.For(0, 0, (x) => Console.WriteLine( "Completed" ) );
}
static void DynamicSchedulingAndWorkStealing()
{
const int iterations = 10000;
var options = new ParallelOptions();// { MaxDegreeOfParallelism = -1 };
var threadsUsed = new ConcurrentDictionary<int, int>();
var sw = new Stopwatch();
sw.Start();
Parallel.ForEach(Enumerable.Range(1, iterations),
options,
() => 0L,
(val, pls, local) =>
{
threadsUsed.TryAdd(Thread.CurrentThread.ManagedThreadId, 0);
if (val < (iterations / 2))
{
//for (int i = 0; i < 1000000; i++) ;
Thread.Sleep(1);
}
return local + 1;
},
local => Console.WriteLine("Thread {0} processed {1} items", Thread.CurrentThread.ManagedThreadId, local));
sw.Stop();
Console.WriteLine("Used {0} different threads and took {1}ms", threadsUsed.Count, sw.ElapsedMilliseconds);
}
/// <summary>
/// Search Asynchrony many extension in all of Fixed and Removable Disks.
/// </summary>
/// <param name="targetExtensions">Some file extensions for use search pattern.</param>
/// <example>
/// FileExtension example:
/// {".jpg", 646546Byte, 646Byte}
/// {".pdf", 25464645546Byte, 60000Byte}
/// </example>
/// <returns>A sorted list of detected files</returns>
public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS)
{
return await Task.Run(() =>
{
searchComplete = false;
//
Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---"));
List<FileInfo> _result = new List<FileInfo>();
//
// Find specific folders from windows drives instead of the total drive.
//
FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation();
//
// Set Data-flow
//
TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir =>
{
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchBeginning,
"Searching {0} ...", dir.FullName));
List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS);
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchCompleted,
"The Search {0} was completed!", dir.FullName));
return res;
}, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount });
ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst));
//
// Search specific folders from windows drives instead of the total drive.
//
try
{
TB.LinkTo(AB);
ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount };
var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir));
TB.Complete();
TB.Completion.Wait();
}
catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); }
searchComplete = true;
Reporter("DiskProbing", new ReportEventArgs("DiskProbing",
ReportCodes.DiskProbingFinished,
"---{Search Disks Finished}---"));
LastSearchResult = _result;
return _result;
});
}
public override HeronValue Eval(VM vm)
{
SeqValue sv = vm.Eval(list) as SeqValue;
if (sv == null)
throw new Exception("Expected list: " + list.ToString());
// internal structure for indexing lists
IInternalIndexable ii = sv.GetIndexable();
if (ii.InternalCount() < 2)
return sv;
HeronValue[] output = new HeronValue[ii.InternalCount()];
ParallelOptions po = new ParallelOptions();
po.MaxDegreeOfParallelism = Config.maxThreads;
var p = Partitioner.Create(1, ii.InternalCount());
HeronValue result = ii.InternalAt(0);
object resultLock = new Object();
Parallel.ForEach(
p,
po,
() =>
{
LoopParams lp = new LoopParams();
lp.op = new OptimizationParams();
lp.acc = lp.op.AddNewAccessor(a);
lp.acc2 = lp.op.AddNewAccessor(b);
lp.vm = vm.Fork();
lp.expr = yield.Optimize(lp.op);
return lp;
},
(Tuple<int, int> range, ParallelLoopState state, LoopParams lp) =>
{
if (range.Item1 == range.Item2)
return lp;
lp.acc.Set(ii.InternalAt(range.Item1));
for (int i = range.Item1 + 1; i < range.Item2; ++i)
{
lp.acc2.Set(ii.InternalAt(i));
lp.acc.Set(lp.vm.Eval(lp.expr));
}
// Update the result
lock (resultLock)
{
lp.acc2.Set(result);
result = lp.vm.Eval(lp.expr);
}
return lp;
},
(LoopParams lp) => { }
);
return new ArrayValue(new HeronValue[] { result }, sv.GetElementType());
}
public void UpdatePreview(/*Vixen.Preview.PreviewElementIntentStates elementStates*/)
{
if (!gdiControl.IsUpdating)
{
IEnumerable<Element> elementArray = VixenSystem.Elements.Where(e => e.State.Any());
if (!elementArray.Any())
{
if (needsUpdate)
{
needsUpdate = false;
gdiControl.BeginUpdate();
gdiControl.EndUpdate();
gdiControl.Invalidate();
}
toolStripStatusFPS.Text = "0 fps";
return;
}
needsUpdate = true;
gdiControl.BeginUpdate();
try
{
var po = new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(elementArray, po, element =>
{
ElementNode node = VixenSystem.Elements.GetElementNodeForElement(element);
if (node != null)
{
List<PreviewPixel> pixels;
if (NodeToPixel.TryGetValue(node, out pixels))
{
foreach (PreviewPixel pixel in pixels)
{
pixel.Draw(gdiControl.FastPixel, element.State);
}
}
}
});
} catch (Exception e)
{
Logging.Error(e.Message, e);
}
gdiControl.EndUpdate();
gdiControl.Invalidate();
toolStripStatusFPS.Text = string.Format("{0} fps", gdiControl.FrameRate);
}
}
public static void H2()
{
string[] values = new string[10000000];
string findValue = " 9000000.";
Console.WriteLine("Building initial array of {0} length...", values.Length);
for (int j = 0; j < values.Length; j++)
{
values[j] = string.Format("This Is My Value: {0}.", j.ToString());
}
start:
Console.WriteLine("Searching for '{0}'...", findValue);
Console.WriteLine("Running parallel search...");
DateTime start = DateTime.Now;
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 4;
System.Threading.Tasks.Parallel.For(0, values.Length, options, (i) =>
{
if (values[i].Contains(findValue))
{
Console.WriteLine("Found {0}", values[i]);
Console.WriteLine("Search completed in: {0}", DateTime.Now - start);
}
});
Console.WriteLine("Parallel for completed in : {0}", DateTime.Now - start);
Console.WriteLine("Running linear search...");
start = DateTime.Now;
for (int i = 0; i < values.Length; i++)
{
if (values[i].Contains(findValue))
{
Console.WriteLine("Found {0}", values[i]);
Console.WriteLine("Search completed in: {0}", DateTime.Now - start);
}
}
Console.WriteLine("Linear for completed in : {0}", DateTime.Now - start);
Console.WriteLine("Press any key to exit or press 'Z' to do it again...");
if (Console.ReadKey().Key == ConsoleKey.Z)
{
Console.WriteLine("");
goto start;
}
}
public DefaultFeatureLexicalDict(int thread_num)
{
featureset_dict_ = new BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
maxid_ = 0;
#if NO_SUPPORT_PARALLEL_LIB
#else
parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = thread_num;
#endif
}
public virtual void Run()
{
processMessageParallerOptions = new ParallelOptions();
processMessageParallerOptions.MaxDegreeOfParallelism = 5;
cancelTokenSource = new CancellationTokenSource();
var token = cancelTokenSource.Token;
var task = Task.Run(() => workCycle(token), token);
task.Wait();
}
public void Cancel()
{
var options = new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount};
Parallel.For(0, _microTasks.Length,
options, i =>
{
Debug.WriteLine("Cancel " + i + ": ");
_microTasks[i].Cancel();
});
}
public AbstractParallelMainProcessor()
{
_tokenSource = new CancellationTokenSource();
_option = new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount - 1,
CancellationToken = _tokenSource.Token
};
this.ParallelMode = true;
}
private static Task ConnectBatch(string url, int batchSize, ConcurrentBag<Connection> connections)
{
var options = new ParallelOptions
{
MaxDegreeOfParallelism = batchSize
};
var tcs = new TaskCompletionSource<object>();
long remaining = batchSize;
Parallel.For(0, batchSize, options, i =>
{
var connection = new Connection(url);
connection.Start().ContinueWith(task =>
{
remaining = Interlocked.Decrement(ref remaining);
if (task.IsFaulted)
{
Console.WriteLine("Failed to start client. {0}", task.Exception.GetBaseException());
}
else
{
connections.Add(connection);
var clientId = connection.ConnectionId;
//connection.Received += data =>
//{
// Console.WriteLine("Client {0} RECEIVED: {1}", clientId, data);
//};
connection.Error += e =>
{
Debug.WriteLine(String.Format("SIGNALR: Client {0} ERROR: {1}", clientId, e));
};
connection.Closed += () =>
{
Debug.WriteLine(String.Format("SIGNALR: Client {0} CLOSED", clientId));
};
}
if (Interlocked.Read(ref remaining) == 0)
{
// When all connections are connected, mark the task as complete
tcs.TrySetResult(null);
}
});
});
return tcs.Task;
}
/// <summary>
/// Execute a Parallel.For loop, but output status messages showing progress every 5 seconds
/// </summary>
/// <param name="Message">The message to output</param>
/// <param name="BeginValue">The lower bound for the for loop, inclusive</param>
/// <param name="EndValue">The upper bound for the for loop, exclusive</param>
/// <param name="Action">The action to execute for each iteration</param>
/// <param name="Log">Log for output messages</param>
public static void ParallelForWithStatus(string Message, int BeginValue, int EndValue, ParallelOptions Options, Action <int> Action, LineBasedTextWriter Log)
{
Log.WriteLine(Message);
ParallelForState State = new ParallelForState();
Parallel.For(BeginValue, EndValue, Options, Index => { Action(Index); State.Increment(EndValue - BeginValue, Message, Log); });
State.Complete(EndValue - BeginValue, Message, Log);
}
public static double[] SolveQr([NotNull][ItemNotNull] this double[][] design, [NotNull] double[] response, [NotNull] ParallelOptions options)
{
if (design is null)
{
throw new ArgumentNullException(nameof(design));
}
if (response is null)
{
throw new ArgumentNullException(nameof(response));
}
double[][] transpose =
design.Transpose(options);
double[][] designTransposeCrossDesign = transpose.MatrixProduct(design, options);
(double[][] orthogonal, double[][] upper) = designTransposeCrossDesign.DecomposeQr();
double[] t = orthogonal.Transpose(options).MatrixProduct(transpose, options).MatrixProduct(response, options);
return(upper.SolveLu(t));
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region All Dashboards
// Set up controller access
string allDashboardsJSON = String.Empty;
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
loggerConsole.Info("All Dashboards");
allDashboardsJSON = controllerApi.GetControllerDashboards();
if (allDashboardsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(allDashboardsJSON, FilePathMap.ControllerDashboards(jobTarget));
}
}
#endregion
#region Dashboards
if (allDashboardsJSON != String.Empty)
{
JArray allDashboardsArray = JArray.Parse(allDashboardsJSON);
loggerConsole.Info("Dashboards ({0} entities)", allDashboardsArray.Count);
int numDashboards = 0;
var listOfDashboardsChunks = allDashboardsArray.BreakListIntoChunks(DASHBOARDS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = DASHBOARDS_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <JToken>, int>(
listOfDashboardsChunks,
parallelOptions,
() => 0,
(listOfDashboardsChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JObject dashboardObject in listOfDashboardsChunk)
{
if (File.Exists(FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"])) == false)
{
string dashboardJSON = controllerApiParallel.GetControllerDashboard((long)dashboardObject["id"]);
if (dashboardJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(dashboardJSON, FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"]));
}
}
}
return(listOfDashboardsChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref numDashboards, finalResult);
Console.Write("[{0}].", numDashboards);
}
);
loggerConsole.Info("Completed {0} Dashboards", allDashboardsArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + allDashboardsArray.Count;
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public string GetImageFileName(int width, int height, double dpiX, double dpiY, CancellationToken cancellationToken, string settingsSummary, string exportDirectory)
{
var sw = Stopwatch.StartNew();
OutputLogEveryXPixel = (width * height / 100);
if (AccelerationStructure)
{
_accelerationStructure = BVHNode.BuildTopDown(_spheres, _logger);
}
var bitmap = new BitmapImage(width, height, dpiX, dpiY);
var divideX = width / (float)2;
var alignX = 1 - divideX;
var divideY = height / (float)2;
var alignY = 1 - divideY;
int workDone = 0;
int totalWork = width * height;
if (Parallelize)
{
var options = new ParallelOptions()
{
CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.For(0, width, options, i =>
{
for (int j = 0; j < height; j++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (AntiAliasing)
{
Vector3 result = Vector3.Zero;
for (int k = 0; k < AntiAliasingSampleSize; k++)
{
var dx = (float)_random.NextGaussian(0d, 0.5d);
var dy = (float)_random.NextGaussian(0d, 0.5d);
var x = (i + alignX + dx) / divideX;
var y = ((height - j) + alignY + dy) / divideY;
var rgb = GetColor(x, y);
result += rgb;
}
var avg_rgb = result * (1f / AntiAliasingSampleSize);
var c = Conversions.FromRGB(avg_rgb, GammaCorrect);
bitmap.Set(i, j, c);
}
else
{
var x = (i + alignX) / divideX;
var y = ((height - j) + alignY) / divideY;
var rgb = GetColor(x, y);
var c = Conversions.FromRGB(rgb, GammaCorrect);
bitmap.Set(i, j, c);
}
var value = Interlocked.Increment(ref workDone);
if (value % OutputLogEveryXPixel == 0)
{
var progress = (float)value / totalWork;
WriteOutput($"{(progress * 100):F3}% progress. Running {sw.Elapsed}. Remaining {TimeSpan.FromMilliseconds(sw.Elapsed.TotalMilliseconds / progress * (1f - progress))}.");
}
}
});
}
else
{
for (int i = 0; i < width; i++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
for (int j = 0; j < height; j++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (AntiAliasing)
{
Vector3 result = Vector3.Zero;
for (int k = 0; k < AntiAliasingSampleSize; k++)
{
var x = (i + alignX + (float)_random.NextGaussian(0d, 0.5d)) / divideX;
var y = ((height - j) + alignY + (float)_random.NextGaussian(0d, 0.5d)) / divideY;
var rgb = GetColor(x, y);
result += rgb;
}
var avg_rgb = result * (1f / AntiAliasingSampleSize);
var c = Conversions.FromRGB(avg_rgb, GammaCorrect);
bitmap.Set(i, j, c);
}
else
{
// Question: Why is it mirrored?
var x = (i + alignX) / divideX;
var y = ((height - j) + alignY) / divideY;
var rgb = GetColor(x, y);
var c = Conversions.FromRGB(rgb, GammaCorrect);
bitmap.Set(i, j, c);
}
++workDone;
if (workDone % OutputLogEveryXPixel == 0)
{
var progress = (float)workDone / totalWork;
WriteOutput($"{(progress * 100):F3}% progress. Running {sw.Elapsed}. Remaining {TimeSpan.FromMilliseconds(sw.Elapsed.TotalMilliseconds / progress * (1f - progress))}.");
}
}
}
}
if (cancellationToken.IsCancellationRequested)
{
WriteOutput("Operation canceled by user.");
return(null);
}
var imageSource = bitmap.GetImageSource();
sw.Stop();
return(SaveImage(imageSource, sw.Elapsed, settingsSummary, exportDirectory));
}
private static Task ConnectBatch(string url, string transport, int batchSize, ConcurrentBag <Connection> connections)
{
var options = new ParallelOptions
{
MaxDegreeOfParallelism = batchSize
};
var batchTcs = new TaskCompletionSource <object>();
long remaining = batchSize;
Parallel.For(0, batchSize, options, async i =>
{
var connection = new Connection(url);
if (!_running)
{
batchTcs.TrySetResult(null);
return;
}
try
{
var clientTransport = GetTransport(transport);
await(clientTransport == null ? connection.Start() : connection.Start(clientTransport));
if (_running)
{
connections.Add(connection);
var clientId = connection.ConnectionId;
connection.Error += e =>
{
Debug.WriteLine(String.Format("SIGNALR: Client {0} ERROR: {1}", clientId, e));
};
connection.Closed += () =>
{
Debug.WriteLine(String.Format("SIGNALR: Client {0} CLOSED", clientId));
// Remove it from the list on close
connections.TryTake(out connection);
};
}
}
catch (Exception ex)
{
Console.WriteLine("Failed to start client. {0}", ex.GetBaseException());
}
finally
{
if (Interlocked.Decrement(ref remaining) == 0)
{
// When all connections are connected, mark the task as complete
batchTcs.TrySetResult(null);
}
}
});
return(batchTcs.Task);
}
/// <summary>
/// Construct with the provided IGenomeDecoder, IPhenomeEvaluator and ParalleOptions.
/// Phenome caching is enabled by default.
/// The number of parallel threads defaults to Environment.ProcessorCount.
/// </summary>
public ParallelGenomeListEvaluator(IGenomeDecoder <TGenome, TPhenome> genomeDecoder,
IPhenomeEvaluator <TPhenome> phenomeEvaluator,
ParallelOptions options)
: this(genomeDecoder, phenomeEvaluator, options, true)
{
}
private void QueryResults()
{
if (_updateSource != null && !_updateSource.IsCancellationRequested)
{
// first condition used for init run
// second condition used when task has already been canceled in last turn
_updateSource.Cancel();
Logger.WoxDebug($"cancel init {_updateSource.Token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {QueryText}");
_updateSource.Dispose();
}
var source = new CancellationTokenSource();
_updateSource = source;
var token = source.Token;
ProgressBarVisibility = Visibility.Hidden;
// support unix-style path separators
var queryText = QueryText.Trim().Replace("/", "\\");
Task.Run(() =>
{
if (!string.IsNullOrEmpty(queryText))
{
if (token.IsCancellationRequested)
{
return;
}
var query = QueryBuilder.Build(queryText, PluginManager.NonGlobalPlugins);
_lastQuery = query;
if (query != null)
{
// handle the exclusiveness of plugin using action keyword
if (token.IsCancellationRequested)
{
return;
}
Task.Delay(200, token).ContinueWith(_ =>
{
Logger.WoxTrace($"progressbar visible 1 {token.GetHashCode()} {token.IsCancellationRequested} {Thread.CurrentThread.ManagedThreadId} {query} {ProgressBarVisibility}");
// start the progress bar if query takes more than 200 ms
if (!token.IsCancellationRequested)
{
ProgressBarVisibility = Visibility.Visible;
}
}, token);
if (token.IsCancellationRequested)
{
return;
}
var plugins = PluginManager.AllPlugins;
var option = new ParallelOptions()
{
CancellationToken = token,
};
CountdownEvent countdown = new CountdownEvent(plugins.Count);
foreach (var plugin in plugins)
{
Task.Run(() =>
{
if (token.IsCancellationRequested)
{
Logger.WoxTrace($"canceled {token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {queryText} {plugin.Metadata.Name}");
countdown.Signal();
return;
}
var results = PluginManager.QueryForPlugin(plugin, query);
if (token.IsCancellationRequested)
{
Logger.WoxTrace($"canceled {token.GetHashCode()} {Thread.CurrentThread.ManagedThreadId} {queryText} {plugin.Metadata.Name}");
countdown.Signal();
return;
}
_resultsQueue.Add(new ResultsForUpdate(results, plugin.Metadata, query, token, countdown));
}, token).ContinueWith(ErrorReporting.UnhandledExceptionHandleTask, TaskContinuationOptions.OnlyOnFaulted);
}
Task.Run(() =>
{
Logger.WoxTrace($"progressbar visible 2 {token.GetHashCode()} {token.IsCancellationRequested} {Thread.CurrentThread.ManagedThreadId} {query} {ProgressBarVisibility}");
// wait all plugins has been processed
try
{
countdown.Wait(token);
}
catch (OperationCanceledException)
{
// todo: why we need hidden here and why progress bar is not working
ProgressBarVisibility = Visibility.Hidden;
return;
}
if (!token.IsCancellationRequested)
{
// used to cancel previous progress bar visible task
source.Cancel();
source.Dispose();
// update to hidden if this is still the current query
ProgressBarVisibility = Visibility.Hidden;
}
});
}
}
else
{
Results.Clear();
Results.Visbility = Visibility.Collapsed;
}
}, token).ContinueWith(ErrorReporting.UnhandledExceptionHandleTask, TaskContinuationOptions.OnlyOnFaulted);
}
public void FailoverThenReconnect()
{
var parallelOptions = new ParallelOptions
{
TaskScheduler = new ThreadPerTaskScheduler(),
MaxDegreeOfParallelism = 100
};
var policy = new ConstantReconnectionPolicy(500);
var nonShareableTestCluster = TestClusterManager.GetNonShareableTestCluster(4, 1, true, false);
using (var cluster = Cluster.Builder().AddContactPoint(nonShareableTestCluster.InitialContactPoint).WithReconnectionPolicy(policy).Build())
{
var session = cluster.Connect();
// Check query to host distribution before killing nodes
var queriedHosts = new List <string>();
DateTime futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
Assert.AreEqual(4, (from singleHost in queriedHosts select singleHost).Distinct().Count(), "All hosts should have been queried!");
// Create list of actions
Action selectAction = () =>
{
var rs = session.Execute("SELECT * FROM system.local");
Assert.Greater(rs.Count(), 0);
};
var actions = new List <Action>();
for (var i = 0; i < 100; i++)
{
actions.Add(selectAction);
//Check that the control connection is using first host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
//Kill some nodes
//Including the one used by the control connection
actions.Insert(20, () => nonShareableTestCluster.Stop(1));
actions.Insert(20, () => nonShareableTestCluster.Stop(2));
actions.Insert(80, () => nonShareableTestCluster.Stop(3));
//Execute in parallel more than 100 actions
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be killed
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "1", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "2", nonShareableTestCluster.Cluster, 20);
TestUtils.WaitForDown(nonShareableTestCluster.ClusterIpPrefix + "3", nonShareableTestCluster.Cluster, 20);
actions = new List <Action>();
for (var j = 0; j < 100; j++)
{
actions.Add(selectAction);
}
//Check that the control connection is using first host
//bring back some nodes
actions.Insert(3, () => nonShareableTestCluster.Start(3));
actions.Insert(50, () => nonShareableTestCluster.Start(2));
actions.Insert(50, () => nonShareableTestCluster.Start(1));
//Execute in parallel more than 100 actions
Trace.TraceInformation("Start invoking with restart nodes");
Parallel.Invoke(parallelOptions, actions.ToArray());
//Wait for the nodes to be restarted
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "1", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "2", DefaultCassandraPort, 30);
TestUtils.WaitForUp(nonShareableTestCluster.ClusterIpPrefix + "3", DefaultCassandraPort, 30);
queriedHosts.Clear();
// keep querying hosts until they are all queried, or time runs out
futureDateTime = DateTime.Now.AddSeconds(120);
while ((from singleHost in queriedHosts select singleHost).Distinct().Count() < 4 && DateTime.Now < futureDateTime)
{
var rs = session.Execute("SELECT * FROM system.local");
queriedHosts.Add(rs.Info.QueriedHost.ToString());
Thread.Sleep(50);
}
//Check that one of the restarted nodes were queried
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "1:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "2:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "3:" + DefaultCassandraPort, queriedHosts);
Assert.Contains(nonShareableTestCluster.ClusterIpPrefix + "4:" + DefaultCassandraPort, queriedHosts);
//Check that the control connection is still using last host
StringAssert.StartsWith(nonShareableTestCluster.ClusterIpPrefix + "4", nonShareableTestCluster.Cluster.Metadata.ControlConnection.Address.ToString());
}
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_BIQ) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ);
return(true);
}
List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration);
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_BIQ)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
stepTimingTarget.NumEntities = entityMetricExtractMappingList.Count;
#endregion
loggerConsole.Info("Extract Metrics for All Entities ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Saved Search
getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, BIQMetric.ENTITY_FOLDER, BIQMetric.ENTITY_TYPE);
#endregion
},
() =>
{
#region Business Journey
getMetricsForEntities(jobTarget, jobConfiguration, entityMetricExtractMappingList, BIQBusinessJourney.ENTITY_FOLDER, BIQBusinessJourney.ENTITY_TYPE);
#endregion
}
);
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
internal void InternalGetInstance(Matrix <string, string, TValue> inputMatrix, ParallelOptions parallelOptions)
{
var selectRowsAndColsView = inputMatrix as SelectRowsAndColsView <string, string, TValue>;
if (null != selectRowsAndColsView && selectRowsAndColsView.ParentMatrix is DenseStructMatrix <TStore, TValue> ) //We optimize this case
{
var parentMatrix = (DenseStructMatrix <TStore, TValue>)selectRowsAndColsView.ParentMatrix;
Parallel.ForEach(RowKeys, parallelOptions, rowKey =>
{
List <TStore> oldStoreList = parentMatrix.RowKeyToStoreList[rowKey];
List <TStore> newStoreList = RowKeyToStoreList[rowKey];
foreach (int oldColIndex in selectRowsAndColsView.IndexOfParentColKey)
{
newStoreList.Add(oldStoreList[oldColIndex]);
}
});
}
else
{
CounterWithMessages counterWithMessages = new CounterWithMessages("Creating new DenseStructMatrix, working on row #{0} of {1}", 1000, RowCount);
Parallel.ForEach(RowKeys, parallelOptions, rowKey =>
{
counterWithMessages.Increment();
foreach (string colKey in ColKeys)
{
TValue value;
if (inputMatrix.TryGetValue(rowKey, colKey, out value))
{
this[rowKey, colKey] = value;
}
}
});
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_WEB);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_WEB);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 3;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region EUM Pages
loggerConsole.Info("Pages and AJAX Requests");
string pagesJSON = controllerApi.GetWEBPages(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (pagesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pagesJSON, FilePathMap.WEBPagesDataFilePath(jobTarget));
}
#endregion
#region EUM Pages Performance
if (pagesJSON != String.Empty)
{
JObject pagesListContainer = JObject.Parse(pagesJSON);
if (isTokenPropertyNull(pagesListContainer, "data") == false)
{
JArray pagesArray = (JArray)pagesListContainer["data"];
loggerConsole.Info("Performance of Pages and Ajax Requests ({0}) entities", pagesArray.Count);
int j = 0;
var listOfPagesChunks = pagesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_PAGES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = PAGES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <JToken>, int>(
listOfPagesChunks,
parallelOptions,
() => 0,
(listOfPagesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JToken pageToken in listOfPagesChunk)
{
string pageName = getStringValueFromJToken(pageToken, "name");
string pageType = getStringValueFromJToken(pageToken, "type");
long pageID = getLongValueFromJToken(pageToken, "addId");
if (File.Exists(FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange)) == false)
{
string pageJSON = controllerApi.GetWEBPagePerformance(jobTarget.ApplicationID, pageID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (pageJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pageJSON, FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange));
}
}
}
return(listOfPagesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Pages", pagesArray.Count);
}
}
#endregion
#region Geo Regions
loggerConsole.Info("Geo Locations");
string geoRegionsJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, String.Empty, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionsJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, "all"));
}
if (geoRegionsJSON != String.Empty)
{
JObject geoRegionsContainerObject = JObject.Parse(geoRegionsJSON);
if (geoRegionsContainerObject != null)
{
if (isTokenPropertyNull(geoRegionsContainerObject, "rumItems") == false)
{
int j = 0;
JArray geoRegionsArray = (JArray)geoRegionsContainerObject["rumItems"];
foreach (JObject geoRegionObject in geoRegionsArray)
{
if (isTokenPropertyNull(geoRegionObject, "eumRegionPerformanceSummaryData") == false)
{
string country = getStringValueFromJToken(geoRegionObject["eumRegionPerformanceSummaryData"], "country");
string geoRegionJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, country, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, country));
}
}
j++;
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
loggerConsole.Info("Completed {0} Geo Locations and Regions", j);
}
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public void MempoolSyncTransactions()
{
using (NodeBuilder builder = NodeBuilder.Create(this))
{
CoreNode stratisNodeSync = builder.CreateStratisPowNode();
CoreNode stratisNode1 = builder.CreateStratisPowNode();
CoreNode stratisNode2 = builder.CreateStratisPowNode();
builder.StartAll();
stratisNodeSync.NotInIBD();
stratisNode1.NotInIBD();
stratisNode2.NotInIBD();
// generate blocks and wait for the downloader to pickup
stratisNodeSync.SetDummyMinerSecret(new BitcoinSecret(new Key(), stratisNodeSync.FullNode.Network));
stratisNodeSync.GenerateStratisWithMiner(105); // coinbase maturity = 100
// wait for block repo for block sync to work
TestHelper.WaitLoop(() => TestHelper.IsNodeSynced(stratisNodeSync));
// sync both nodes
stratisNode1.CreateRPCClient().AddNode(stratisNodeSync.Endpoint, true);
stratisNode2.CreateRPCClient().AddNode(stratisNodeSync.Endpoint, true);
TestHelper.WaitLoop(() => TestHelper.AreNodesSynced(stratisNode1, stratisNodeSync));
TestHelper.WaitLoop(() => TestHelper.AreNodesSynced(stratisNode2, stratisNodeSync));
// create some transactions and push them to the pool
var trxs = new List <Transaction>();
foreach (int index in Enumerable.Range(1, 5))
{
Block block = stratisNodeSync.FullNode.BlockStoreManager().BlockRepository.GetAsync(stratisNodeSync.FullNode.Chain.GetBlock(index).HashBlock).Result;
Transaction prevTrx = block.Transactions.First();
var dest = new BitcoinSecret(new Key(), stratisNodeSync.FullNode.Network);
Transaction tx = stratisNodeSync.FullNode.Network.CreateTransaction();
tx.AddInput(new TxIn(new OutPoint(prevTrx.GetHash(), 0), PayToPubkeyHashTemplate.Instance.GenerateScriptPubKey(stratisNodeSync.MinerSecret.PubKey)));
tx.AddOutput(new TxOut("25", dest.PubKey.Hash));
tx.AddOutput(new TxOut("24", new Key().PubKey.Hash)); // 1 btc fee
tx.Sign(stratisNodeSync.FullNode.Network, stratisNodeSync.MinerSecret, false);
trxs.Add(tx);
}
var options = new ParallelOptions {
MaxDegreeOfParallelism = 5
};
Parallel.ForEach(trxs, options, transaction =>
{
stratisNodeSync.Broadcast(transaction);
});
// wait for all nodes to have all trx
TestHelper.WaitLoop(() => stratisNodeSync.CreateRPCClient().GetRawMempool().Length == 5);
// the full node should be connected to both nodes
Assert.True(stratisNodeSync.FullNode.ConnectionManager.ConnectedPeers.Count() >= 2);
TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetRawMempool().Length == 5);
TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetRawMempool().Length == 5);
// mine the transactions in the mempool
stratisNodeSync.GenerateStratisWithMiner(1);
TestHelper.WaitLoop(() => stratisNodeSync.CreateRPCClient().GetRawMempool().Length == 0);
// wait for block and mempool to change
TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetBestBlockHash() == stratisNodeSync.CreateRPCClient().GetBestBlockHash());
TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetBestBlockHash() == stratisNodeSync.CreateRPCClient().GetBestBlockHash());
TestHelper.WaitLoop(() => stratisNode1.CreateRPCClient().GetRawMempool().Length == 0);
TestHelper.WaitLoop(() => stratisNode2.CreateRPCClient().GetRawMempool().Length == 0);
}
}
private void Form1_DragDrop(object sender, DragEventArgs e)
{
return; //Not added yet
string[] files = (string[])e.Data.GetData(DataFormats.FileDrop);
if (Path.GetExtension(files[0]).ToUpper() == ".WIL" ||
Path.GetExtension(files[0]).ToUpper() == ".WZL" ||
Path.GetExtension(files[0]).ToUpper() == ".MIZ")
{
try
{
ParallelOptions options = new ParallelOptions {
MaxDegreeOfParallelism = 8
};
Parallel.For(0, files.Length, options, i =>
{
if (Path.GetExtension(files[i]) == ".wtl")
{
WTLLibrary WTLlib = new WTLLibrary(files[i]);
WTLlib.ToMLibrary();
}
else
{
WeMadeLibrary WILlib = new WeMadeLibrary(files[i]);
WILlib.ToMLibrary();
}
toolStripProgressBar.Value++;
});
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
toolStripProgressBar.Value = 0;
MessageBox.Show(
string.Format("Successfully converted {0} {1}",
(OpenWeMadeDialog.FileNames.Length).ToString(),
(OpenWeMadeDialog.FileNames.Length > 1) ? "libraries" : "library"));
}
else if (Path.GetExtension(files[0]).ToUpper() == ".LIB")
{
ClearInterface();
ImageList.Images.Clear();
PreviewListView.Items.Clear();
_indexList.Clear();
if (_library != null)
{
_library.Close();
}
//_library = new MLibraryV2(files[0]);
//PreviewListView.VirtualListSize = _library.Images.Count;
PreviewListView.RedrawItems(0, PreviewListView.Items.Count - 1, true);
// Show .Lib path in application title.
this.Text = files[0].ToString();
}
else
{
return;
}
}
public ActionResult BacktesterResults(string exchange, string coinsToBuy, string baseCurrency, bool saveSignals, decimal startingWallet, decimal tradeAmount, DateTime?from = null, DateTime?to = null, string candleSize = "5", string strategy = "all")
{
var strategies = new JObject();
var coins = new List <string>();
if (String.IsNullOrEmpty(coinsToBuy))
{
IExchangeAPI api = ExchangeAPI.GetExchangeAPI(exchange);
var exchangeCoins = api.GetMarketSymbolsMetadataAsync().Result.Where(m => m.BaseCurrency == baseCurrency);
foreach (var coin in exchangeCoins)
{
coins.Add(api.ExchangeMarketSymbolToGlobalMarketSymbolAsync(coin.MarketSymbol).Result);
}
}
else
{
Char delimiter = ',';
String[] coinsToBuyArray = coinsToBuy.Split(delimiter);
foreach (var coin in coinsToBuyArray)
{
coins.Add(coin.ToUpper());
}
}
var backtestOptions = new BacktestOptions
{
DataFolder = Global.DataPath,
Exchange = (Exchange)Enum.Parse(typeof(Exchange), exchange, true),
Coins = coins,
CandlePeriod = Int32.Parse(candleSize)
};
if (from.HasValue)
{
backtestOptions.StartDate = from.Value;
}
if (to.HasValue)
{
backtestOptions.EndDate = to.Value;
backtestOptions.EndDate = backtestOptions.EndDate.AddDays(1).AddMinutes(-1);
}
if (tradeAmount == 0m)
{
tradeAmount = backtestOptions.StakeAmount;
}
if (startingWallet == 0m)
{
startingWallet = backtestOptions.StartingWallet;
}
var cts = new CancellationTokenSource();
var parallelOptions = new ParallelOptions
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(BacktestFunctions.GetTradingStrategies(), parallelOptions, async tradingStrategy =>
{
if (strategy != "all")
{
var base64EncodedBytes = Convert.FromBase64String(strategy);
if (tradingStrategy.Name != Encoding.UTF8.GetString(base64EncodedBytes))
{
return;
}
}
var result = await BacktestFunctions.BackTestJson(tradingStrategy, backtestOptions, Global.DataStoreBacktest, baseCurrency, saveSignals, startingWallet, tradeAmount);
for (int i = 0; i < result.Count(); i++)
{
if (i == 0)
{
await Runtime.GlobalHubBacktest.Clients.All.SendAsync("SendSummary", JsonConvert.SerializeObject(result[i]));
}
else
{
await Runtime.GlobalHubBacktest.Clients.All.SendAsync("Send", JsonConvert.SerializeObject(result[i]));
}
}
});
return(new JsonResult(strategies));
}
bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args)
{
Console.WriteLine("Running encoding process...");
var old_obj = double.MaxValue;
var converge = 0;
var lbfgs = new LBFGS(args.ThreadsNum);
lbfgs.expected = new double[modelWriter.feature_size() + 1];
var processList = new List <CRFEncoderThread>();
var parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = args.ThreadsNum;
//Initialize encoding threads
for (var i = 0; i < args.ThreadsNum; i++)
{
var thread = new CRFEncoderThread();
thread.start_i = i;
thread.thread_num = args.ThreadsNum;
thread.x = x;
thread.lbfgs = lbfgs;
thread.Init();
processList.Add(thread);
}
//Statistic term and result tags frequency
var termNum = 0;
int[] yfreq;
yfreq = new int[modelWriter.y_.Count];
for (int index = 0; index < x.Length; index++)
{
var tagger = x[index];
termNum += tagger.word_num;
for (var j = 0; j < tagger.word_num; j++)
{
yfreq[tagger.answer_[j]]++;
}
}
//Iterative training
var startDT = DateTime.Now;
var dMinErrRecord = 1.0;
for (var itr = 0; itr < args.MaxIteration; ++itr)
{
//Clear result container
lbfgs.obj = 0.0f;
lbfgs.err = 0;
lbfgs.zeroone = 0;
Array.Clear(lbfgs.expected, 0, lbfgs.expected.Length);
var threadList = new List <Thread>();
for (var i = 0; i < args.ThreadsNum; i++)
{
var thread = new Thread(processList[i].Run);
thread.Start();
threadList.Add(thread);
}
int[,] merr;
merr = new int[modelWriter.y_.Count, modelWriter.y_.Count];
for (var i = 0; i < args.ThreadsNum; ++i)
{
threadList[i].Join();
lbfgs.obj += processList[i].obj;
lbfgs.err += processList[i].err;
lbfgs.zeroone += processList[i].zeroone;
Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}");
Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}");
//Calculate error
for (var j = 0; j < modelWriter.y_.Count; j++)
{
for (var k = 0; k < modelWriter.y_.Count; k++)
{
merr[j, k] += processList[i].merr[j, k];
}
}
}
long num_nonzero = 0;
var fsize = modelWriter.feature_size();
var alpha = modelWriter.alpha_;
if (orthant == true)
{
//L1 regularization
Parallel.For <double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
{
subtotal += Math.Abs(alpha[k] / modelWriter.cost_factor_);
if (alpha[k] != 0.0)
{
Interlocked.Increment(ref num_nonzero);
}
return(subtotal);
},
(subtotal) => // lock free accumulator
{
double initialValue;
double newValue;
do
{
initialValue = lbfgs.obj; // read current value
newValue = initialValue + subtotal; //calculate new value
}while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
});
}
else
{
//L2 regularization
num_nonzero = fsize;
Parallel.For <double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
{
subtotal += (alpha[k] * alpha[k] / (2.0 * modelWriter.cost_factor_));
lbfgs.expected[k] += (alpha[k] / modelWriter.cost_factor_);
return(subtotal);
},
(subtotal) => // lock free accumulator
{
double initialValue;
double newValue;
do
{
initialValue = lbfgs.obj; // read current value
newValue = initialValue + subtotal; //calculate new value
}while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
});
}
//Show each iteration result
var diff = (itr == 0 ? 1.0f : Math.Abs(old_obj - lbfgs.obj) / old_obj);
old_obj = lbfgs.obj;
ShowEvaluation(x.Length, modelWriter, lbfgs, termNum, itr, merr, yfreq, diff, startDT, num_nonzero, args);
if (diff < args.MinDifference)
{
converge++;
}
else
{
converge = 0;
}
if (itr > args.MaxIteration || converge == 3)
{
break; // 3 is ad-hoc
}
if (args.DebugLevel > 0 && (double)lbfgs.zeroone / (double)x.Length < dMinErrRecord)
{
var cc = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("[Debug Mode] ");
Console.ForegroundColor = cc;
//Save current best feature weight into file
dMinErrRecord = (double)lbfgs.zeroone / (double)x.Length;
modelWriter.SaveFeatureWeight("feature_weight_tmp", false);
}
int iret;
iret = lbfgs.optimize(alpha, modelWriter.cost_factor_, orthant);
if (iret <= 0)
{
return(false);
}
}
Console.WriteLine("Completed encoding process.");
return(true);
}
static void Main(string[] args)
{
Console.WriteLine("Please enter your password: "******"%4@";
object Locker = new object();
Stopwatch watch1 = new Stopwatch();
watch1.Start();
//List<char[]> AllPossible = new List<char[]>();
CancellationTokenSource cts = new CancellationTokenSource();
ParallelOptions Po = new ParallelOptions();
Po.CancellationToken = cts.Token;
Po.MaxDegreeOfParallelism = Environment.ProcessorCount;
Task.Factory.StartNew(() =>
{
if (char.IsLetterOrDigit(Console.ReadKey().KeyChar))
{
cts.Cancel();
}
});
try
{
Parallel.ForEach(result, Po, (item, state) =>
{
//lock (Locker)
//{
bool validate = true;
for (int i = 0; i < InputPassCode.Length; i++)
{
if (item[i] != InputPassCode[i])
{
validate = false;
break;
}
}
Console.Write(item);
if (validate == true)
{
Console.WriteLine(item);
state.Break();
}
Po.CancellationToken.ThrowIfCancellationRequested();
//}
//lock (AllPossible)
//{
// AllPossible.Add(item);
//}
});
}
catch (OperationCanceledException e)
{
Console.WriteLine($"\n{e.Message}");
}
finally
{
cts.Dispose();
}
//foreach (var item in result)
//{
// bool validate = true;
// for (int i = 0; i < InputPassCode.Length; i++)
// {
// if (item[i] != InputPassCode[i])
// {
// validate = false;
// break;
// }
// }
// if (validate == true)
// {
// Console.WriteLine(item);
// break;
// }
//}
watch1.Stop();
Console.WriteLine($"\n{watch.Elapsed}");
Console.WriteLine(watch1.Elapsed);
//Console.WriteLine($"Items in list = {PassCodeBank.Count}");
//Console.WriteLine(result.Count());
}
bool Decode(CRFSharpWrapper.DecoderArgs options)
{
var parallelOption = new ParallelOptions();
if (File.Exists(options.strInputFileName) == false)
{
Logger.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName);
return(false);
}
if (File.Exists(options.strModelFileName) == false)
{
Logger.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName);
return(false);
}
var sr = new StreamReader(options.strInputFileName);
StreamWriter sw = null, swSeg = null;
if (options.strOutputFileName != null && options.strOutputFileName.Length > 0)
{
sw = new StreamWriter(options.strOutputFileName);
}
if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0)
{
swSeg = new StreamWriter(options.strOutputSegFileName);
}
//Create CRFSharp wrapper instance. It's a global instance
var crfWrapper = new CRFSharpWrapper.Decoder();
//Load encoded model from file
Logger.WriteLine("Loading model from {0}", options.strModelFileName);
crfWrapper.LoadModel(options.strModelFileName);
var queueRecords = new ConcurrentQueue <List <List <string> > >();
var queueSegRecords = new ConcurrentQueue <List <List <string> > >();
parallelOption.MaxDegreeOfParallelism = options.thread;
Parallel.For(0, options.thread, parallelOption, t =>
{
//Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
var tagger = crfWrapper.CreateTagger(options.nBest, options.maxword);
tagger.set_vlevel(options.probLevel);
//Initialize result
var crf_out = new crf_seg_out[options.nBest];
for (var i = 0; i < options.nBest; i++)
{
crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
}
var inbuf = new List <List <string> >();
while (true)
{
lock (rdLocker)
{
if (ReadRecord(inbuf, sr) == false)
{
break;
}
queueRecords.Enqueue(inbuf);
queueSegRecords.Enqueue(inbuf);
}
//Call CRFSharp wrapper to predict given string's tags
if (swSeg != null)
{
crfWrapper.Segment(crf_out, tagger, inbuf);
}
else
{
crfWrapper.Segment((crf_term_out[])crf_out, (DecoderTagger)tagger, inbuf);
}
List <List <string> > peek = null;
//Save segmented tagged result into file
if (swSeg != null)
{
var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
while (peek != inbuf)
{
queueSegRecords.TryPeek(out peek);
}
for (int index = 0; index < rstList.Count; index++)
{
var item = rstList[index];
swSeg.WriteLine(item);
}
queueSegRecords.TryDequeue(out peek);
peek = null;
}
//Save raw tagged result (with probability) into file
if (sw != null)
{
while (peek != inbuf)
{
queueRecords.TryPeek(out peek);
}
OutputRawResultToFile(inbuf, crf_out, tagger, sw);
queueRecords.TryDequeue(out peek);
}
}
});
sr.Close();
if (sw != null)
{
sw.Close();
}
if (swSeg != null)
{
swSeg.Close();
}
return(true);
}
private async void btnScrape_Click(object sender, EventArgs e)
{
btnScrape.Enabled = false;
var hosts = new List <string>();
if (rbCustom.Checked)
{
if (CustomSources.Count == 0)
{
MessageBox.Show("You have selected custom source list. Please load some before scraping.", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information);
return;
}
hosts.Clear();
hosts.AddRange(CustomSources.ToArray());
}
// hosts.Add("https://orca.tech/?action=real-time-proxy-list");
//hosts.Add("http://free-proxy-list.net/anonymous-proxy.html");
// hosts.Add("http://www.us-proxy.org/");
// hosts.Add("www.sslproxies.org");
//hosts.Add("http://irc-proxies24.blogspot.com/2016/08/26-08-16-irc-proxy-servers-900_26.html");
// hosts.Add("http://www.samair.ru/proxy/");
//hosts.Add("https://www.hide-my-ip.com/proxylist.shtml");
//hosts.Add("http://fineproxy.org/eng/?p=6");
//hosts.Add("http://www.blackhatworld.com/seo/new-fresh-big-proxy-lists-worldwide-usa-and-elite-proxies-updated-daily.753956/page-21");
//hosts.Add("https://us-proxy-server.blogspot.com/");
// hosts.Add("http://txt.proxyspy.net/proxy.txt");
//hosts.Add("http://txt.proxyspy.net/proxy.txt");
// hosts.Add("http://proxyrox.com");
//hosts.Add("https://nordvpn.com/wp-admin/admin-ajax.php?searchParameters[0][name]=proxy-country&searchParameters[0][value]=&searchParameters[1][name]=proxy-ports&searchParameters[1][value]=&offset=25&limit=10000&action=getProxies");
lvProxies.BeginUpdate();
// BLOGSPOT
//hosts.Add("http://proxyserverlist-24.blogspot.com/");
//hosts.Add("http://sslproxies24.blogspot.ro");
// hosts.Add("http://sslproxies24.blogspot.ro");
bool checkLimit = cbLimit.Checked;
var numLimit = (int)this.numLimit.Value;
var options = new ParallelOptions()
{
MaxDegreeOfParallelism = 10
};
var _Scraper = new Scraper.Scraper();
Hashtable hash = new Hashtable();
Stopwatch s = new Stopwatch();
s.Start();
await Task.Run(() =>
{
Parallel.ForEach(hosts, options, (item) =>
{
try
{
if (checkLimit && hash.Count >= numLimit)
{
return;
}
if (!item.StartsWith("http://") && !item.StartsWith("https://"))
{
item = "http://" + item;
}
string html = HTTP.DoWebRequest(item);
if (string.IsNullOrEmpty(html))
{
return;
}
List <Proxy> proxies = _Scraper.Scrape(item, html);
if (proxies == null)
{
return;
}
Parallel.ForEach(proxies, options, (proxy) =>
{
if (proxy == null)
{
return;
}
if (checkLimit && hash.Count >= numLimit)
{
return;
}
lock (hash)
{
if (!hash.Contains(proxy.Proxy_))
{
hash.Add(proxy.Proxy_, proxy);
}
}
});
}
catch { }
});
});
foreach (DictionaryEntry element in hash)
{
if (checkLimit && lvProxies.Items.Count >= numLimit)
{
break;
}
Proxy proxy = (Proxy)(element.Value);
Invoke(new MethodInvoker(() =>
{
ListViewItem i = new ListViewItem((lvProxies.Items.Count + 1).ToString());
var countryCode = CountryInfo.GetCode(proxy.Country);
if (!imageList.Images.Keys.Contains(countryCode))
{
imageList.Images.Add(countryCode, Image.FromFile(@"Flags\" + countryCode + ".png"));
}
i.ImageKey = countryCode;
// i.UseItemStyleForSubItems = false;
i.SubItems.Add(proxy.Proxy_);
i.SubItems.Add(proxy.Anonymity);
i.SubItems.Add(proxy.Country);
i.SubItems.Add("");
i.SubItems.Add("");
i.SubItems.Add("");
lvProxies.Items.Add(i);
}));
}
s.Stop();
lvProxies.EndUpdate();
MessageBox.Show("Done!\r\nTime Elapsed: " + s.Elapsed);
btnScrape.Enabled = true;
}
private void InternalSearch(CancellationToken cancelToken, PauseTokenSource pauseTokenSource)
{
ElapsedTimer.Start();
SearchSW.Start();
var duplicateDict = new Dictionary <string, DuplicateItem>();
try {
var parallelOpts = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = cancelToken
};
var reScanList = ScanFileList
.Where(vf => !vf.Flags.Any(EntryFlags.ManuallyExcluded | EntryFlags.AllErrors))
.Where(vf => (vf.mediaInfo == null && !vf.IsImage) || vf.grayBytes == null || vf.grayBytes.Count != Settings.ThumbnailCount)
.ToList();
InitProgress(reScanList.Count);
Parallel.For(0, reScanList.Count, parallelOpts, i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
var entry = reScanList[i];
if (entry.mediaInfo == null && !entry.IsImage)
{
var ffProbe = new FFProbeWrapper.FFProbeWrapper();
var info = ffProbe.GetMediaInfo(entry.Path);
if (info == null)
{
entry.Flags.Set(EntryFlags.MetadataError);
return;
}
entry.mediaInfo = info;
}
if (entry.grayBytes == null)
{
var(error, grayBytes) = entry.IsImage ? GetImageAsBitmaps(entry, positionList.Count) : GetVideoThumbnailAsBitmaps(entry, positionList);
if (error > 0)
{
entry.Flags.Set(error);
}
else
{
entry.grayBytes = grayBytes;
}
}
IncrementProgress(entry.Path);
});
SearchSW.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.ThumbnailsFinished, SearchSW.Elapsed, processedFiles));
SearchSW.Restart();
var percentageDifference = 1.0f - Settings.Percent / 100f;
var dupeScanList = ScanFileList.Where(vf => !vf.Flags.Any(EntryFlags.AllErrors | EntryFlags.ManuallyExcluded)).ToList();
InitProgress(dupeScanList.Count);
Parallel.For(0, dupeScanList.Count, parallelOpts, i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
var baseItem = dupeScanList[i];
if (baseItem.grayBytes == null || baseItem.grayBytes.Count == 0)
{
IncrementProgress(baseItem.Path);
return;
}
for (var n = i + 1; n < dupeScanList.Count; n++)
{
var compItem = dupeScanList[n];
if (baseItem.IsImage && !compItem.IsImage)
{
continue;
}
if (compItem.grayBytes == null || compItem.grayBytes.Count == 0)
{
continue;
}
if (baseItem.grayBytes.Count != compItem.grayBytes.Count)
{
continue;
}
var duplicateCounter = 0;
var percent = new float[baseItem.grayBytes.Count];
for (var j = 0; j < baseItem.grayBytes.Count; j++)
{
Debug.Assert(baseItem.grayBytes[j].Length == compItem.grayBytes[j].Length, "Images must be of same length");
percent[j] = ExtensionMethods.PercentageDifference(baseItem.grayBytes[j], compItem.grayBytes[j]);
if (percent[j] < percentageDifference)
{
duplicateCounter++;
}
else
{
break;
}
}
if (duplicateCounter != baseItem.grayBytes.Count)
{
continue;
}
var percSame = percent.Average();
lock (duplicateDict) {
var foundBase = duplicateDict.TryGetValue(baseItem.Path, out var existingBase);
var foundComp = duplicateDict.TryGetValue(compItem.Path, out var existingComp);
if (foundBase && foundComp)
{
//this happens with 4+ identical items:
//first, 2+ duplicate groups are found independently, they are merged in this branch
if (existingBase.GroupId != existingComp.GroupId)
{
foreach (var dup in duplicateDict.Values.Where(c => c.GroupId == existingComp.GroupId))
{
dup.GroupId = existingBase.GroupId;
}
}
}
else if (foundBase)
{
duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame)
{
GroupId = existingBase.GroupId
});
}
else if (foundComp)
{
duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame)
{
GroupId = existingComp.GroupId
});
}
else
{
var groupId = Guid.NewGuid();
duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame)
{
GroupId = groupId
});
duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame)
{
GroupId = groupId
});
}
}
}
IncrementProgress(baseItem.Path);
});
SearchSW.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.DuplicatesCheckFinishedIn, SearchSW.Elapsed));
Duplicates = new HashSet <DuplicateItem>(duplicateDict.Values);
}
catch (OperationCanceledException) {
Logger.Instance.Info(Properties.Resources.CancellationExceptionCaught);
}
}
private async void btnCheck_Click(object sender, EventArgs e)
{
if (lvProxies.Items.Count <= 0)
{
return;
}
// Validity checks
if (cbSSL.Checked)
{
if (!tbSSL.Text.StartsWith("https://"))
{
MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information);
return;
}
try
{
var url = new Uri(tbSSL.Text);
}
catch
{
MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information);
return;
}
}
if (cbCustom.Checked)
{
//if (!tbCustom.Text.StartsWith("https://"))
//{
// MessageBox.Show("Please enter a valid HTTPS url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information);
// return;
//}
try
{
var url = new Uri(tbCustom.Text);
}
catch
{
MessageBox.Show("Please enter a valid custom url to test!", "Form Validation Failed", MessageBoxButtons.OK, MessageBoxIcon.Information);
return;
}
}
//UpdateStatus("Checking proxies...");
btnCheck.Enabled = false;
proxyChecked = false;
// cmsRotator.Enabled = false;
// btnRotate.Enabled = false;
var proxies = new List <ListViewItem>();
foreach (ListViewItem item in lvProxies.Items)
{
proxies.Add(item);
}
var options = new ParallelOptions {
MaxDegreeOfParallelism = 50
};
await Task.Run(() =>
{
try
{
Parallel.ForEach(proxies, options, proxy =>
{
string[] prox = proxy.SubItems[1].Text.Split(new[] { ":" }, StringSplitOptions.None);
ProxyResult r = Checker.Checker.Check(
new WebProxy(prox[0], int.Parse(prox[1])),
int.Parse(prox[1]),
cbSSL.Checked ? tbSSL.Text : "",
cbCustom.Checked ? tbCustom.Text : ""
);
Invoke(new MethodInvoker(() =>
{
if (!r.Working || r.Speed > 10000)
{
//if (dead || (cbSlow.Checked && speed > 10000))
lvProxies.Items[proxy.Index].BackColor = Color.FromArgb(255, 207, 206);
}
else
{
lvProxies.Items[proxy.Index].SubItems[2].Text = r.Anonymity;
lvProxies.Items[proxy.Index].SubItems[3].Text = r.CountryInfo[1];
var countryCode = CountryInfo.GetCode(r.CountryInfo[1]);
if (!imageList.Images.Keys.Contains(countryCode))
{
imageList.Images.Add(countryCode, Image.FromFile(@"Flags\" + countryCode + ".png"));
}
lvProxies.Items[proxy.Index].ImageKey = countryCode;
lvProxies.Items[proxy.Index].SubItems[4].Text = r.Speed + " ms";
if (cbSSL.Checked)
{
lvProxies.Items[proxy.Index].SubItems[5].Text = r.SSL ? "Yes" : "No";
lvProxies.Items[proxy.Index].SubItems[5].ForeColor = r.SSL ? Color.DarkGreen : Color.Red;
}
else
{
lvProxies.Items[proxy.Index].SubItems[5].Text = "N\\A";
}
if (cbCustom.Checked)
{
lvProxies.Items[proxy.Index].SubItems[6].Text = r.Custom ? "Yes" : "No";
lvProxies.Items[proxy.Index].SubItems[6].ForeColor = r.Custom ? Color.DarkGreen : Color.Red;
}
else
{
lvProxies.Items[proxy.Index].SubItems[6].Text = "N\\A";
}
lvProxies.Items[proxy.Index].BackColor = Color.FromArgb(202, 255, 202);
}
}));
});
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
});
//lProxyCount.Text = "Loaded Proxies: " + lvProxies.Items.Count;
//UpdateStatus("Finished checking proxies.");
btnCheck.Enabled = true;
proxyChecked = true;
//cmsRotator.Enabled = true;
//btnRotate.Enabled = true;
}
public void Run(string[] args)
{
if (args.Length == 0)
{
var dialog = new MainDiscoForm();
dialog.ShowDialog();
return;
}
bool scanCues = false;
string dirArg = null;
string infile = null;
var loadDiscInterface = DiscInterface.BizHawk;
var compareDiscInterfaces = new List <DiscInterface>();
bool hawk = false;
int idx = 0;
while (idx < args.Length)
{
string a = args[idx++];
string au = a.ToUpperInvariant();
if (au == "LOAD")
{
loadDiscInterface = (DiscInterface)Enum.Parse(typeof(DiscInterface), args[idx++], true);
}
else if (au == "COMPARE")
{
compareDiscInterfaces.Add((DiscInterface)Enum.Parse(typeof(DiscInterface), args[idx++], true));
}
else if (au == "HAWK")
{
hawk = true;
}
else if (au == "CUEDIR")
{
dirArg = args[idx++];
scanCues = true;
}
else
{
infile = a;
}
}
if (hawk)
{
if (infile == null)
{
return;
}
// TODO - write it out
var dmj = new DiscMountJob {
IN_DiscInterface = loadDiscInterface, IN_FromPath = infile
};
dmj.Run();
}
bool verbose = true;
if (scanCues)
{
verbose = false;
var todo = FindCuesRecurse(dirArg);
var po = new ParallelOptions();
var cts = new CancellationTokenSource();
po.CancellationToken = cts.Token;
po.MaxDegreeOfParallelism = 1;
if (po.MaxDegreeOfParallelism < 0)
{
po.MaxDegreeOfParallelism = 1;
}
object olock = new object();
int ctr = 0;
bool blocked = false;
try
{
Parallel.ForEach(todo, po, (fp) =>
{
lock (olock)
{
ctr++;
int strlen = todo.Count.ToString().Length;
string fmt = string.Format("{{0,{0}}}/{{1,{0}}} {{2}}", strlen);
Console.WriteLine(fmt, ctr, todo.Count, Path.GetFileNameWithoutExtension(fp));
}
if (!blocked)
{
foreach (var cmpif in compareDiscInterfaces)
{
var sw = new StringWriter();
bool success = CompareFile(fp, loadDiscInterface, cmpif, verbose, cts, sw);
if (!success)
{
lock (Console.Out)
Console.Out.Write(sw.ToString());
cts.Cancel();
return;
}
}
}
});
}
catch (AggregateException ae) {
Console.WriteLine(ae.ToString());
}
catch (OperationCanceledException oce)
{
Console.WriteLine(oce.ToString());
}
Console.WriteLine("--TERMINATED--");
return;
}
if (compareDiscInterfaces.Count != 0)
{
var sw = new StringWriter();
foreach (var cmpif in compareDiscInterfaces)
{
CompareFile(infile, loadDiscInterface, cmpif, verbose, null, sw);
}
sw.Flush();
string results = sw.ToString();
var cr = new ComparisonResults {
textBox1 = { Text = results }
};
cr.ShowDialog();
}
} //Run()
/// <summary>
/// 前九字段查询
/// </summary>
/// <param name="index"></param>
/// <param name="pageSize"></param>
/// <returns></returns>
public async Task <ResponseModel <t2_house_part_expand> > GetJsonHousePart(int index, int pageSize)
{
//string sql;
//if (index > 1)
//{
// sql = $"select top {pageSize} * from t2_house where id > (select max(id) from (select top(({index} - 1) * {pageSize}) id from t2_house order by id) as T) order by id";
//}
//else
//{
// sql = $"select top {pageSize} * from t2_house";
//}
string sql = $"select jsonstr from t2_house order by id offset {pageSize * (index - 1)} row fetch next {pageSize} rows only";
//string sql = $"select top {pageSize} jsonstr from t2_house where id >(select max(id) from (select id from t2_house order by id offset ({pageSize} * ({index} - 1)) row fetch next {pageSize} rows only) as T) order by id";
ConcurrentBag <t2_house_part_expand> t2modelList = new ConcurrentBag <t2_house_part_expand>();
var result = await dataService.GetListAsync <t2_house>(sql);
ParallelOptions opt = new ParallelOptions
{
MaxDegreeOfParallelism = 2
};
//string data = "Data Source=192.168.0.89 ; Initial Catalog = PressureTest; Persist Security Info=True; User ID = sa; Password=qj12345678@";
//ConcurrentBag<t2_house> cb = new ConcurrentBag<t2_house>();
//using (SqlConnection conn = new SqlConnection(data))
//{
// conn.Open();
// string sql = $"select top {pageSize} jsonstr from t2_house where id >(select max(id) from (select id from t2_house order by id offset ({pageSize} * ({index} - 2)) row fetch next {pageSize} rows only) as T) order by id";
// SqlCommand command = new SqlCommand(sql, conn);
// DataTable dt = new DataTable();
// SqlDataAdapter da = new SqlDataAdapter(command);
// da.Fill(dt);
// Parallel.For(0, dt.Rows.Count, i =>
// {
// Parallel.Invoke(() =>
// {
// cb.Add(new t2_house { jsonstr = dt.Rows[i][0].ToString() });
// });
// });
//}
Parallel.ForEach(result, opt, item =>
{
var model = JsonConvert.DeserializeObject <t2_house_expand>(item.jsonstr);
t2_house_part_expand modeltemp = new t2_house_part_expand();
Parallel.Invoke(() =>
{
T2_ModifyLogModel column4 = model.column4.OrderByDescending(a => a.Column207).FirstOrDefault();
T2_ModifyLogModel column5 = model.column5.OrderByDescending(a => a.Column207).FirstOrDefault();
T2_ModifyLogModel column6 = model.column6.OrderByDescending(a => a.Column207).FirstOrDefault();
T2_ModifyLogModel column7 = model.column7.OrderByDescending(a => a.Column207).FirstOrDefault();
T2_ModifyLogModel column8 = model.column8.OrderByDescending(a => a.Column207).FirstOrDefault();
BaseModel.Mapper(modeltemp, model);
modeltemp.column4 = column4.Column205;
modeltemp.column5 = column5.Column205;
modeltemp.column6 = column6.Column205;
modeltemp.column7 = column7.Column205;
modeltemp.column8 = column8.Column205;
t2modelList.Add(modeltemp);
});
});
ResponseModel <t2_house_part_expand> resModel = new ResponseModel <t2_house_part_expand>(t2modelList.ToList().OrderByDescending(o => o.column1).ToList());
return(resModel);
}
static void CheckMD5(ProgressReporterDialogue frmProgressReporter, string url)
{
var baseurl = ConfigurationManager.AppSettings["UpdateLocation"];
if (dobeta)
{
baseurl = ConfigurationManager.AppSettings["BetaUpdateLocation"];
}
L10N.ReplaceMirrorUrl(ref baseurl);
string responseFromServer = "";
WebRequest request = WebRequest.Create(url);
request.Timeout = 10000;
// Set the Method property of the request to POST.
request.Method = "GET";
// Get the response.
// Get the stream containing content returned by the server.
// Open the stream using a StreamReader for easy access.
using (WebResponse response = request.GetResponse())
using (Stream dataStream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(dataStream))
{
// Display the status.
log.Info(((HttpWebResponse)response).StatusDescription);
// Read the content.
responseFromServer = reader.ReadToEnd();
}
Regex regex = new Regex(@"([^\s]+)\s+upgrade/(.*)", RegexOptions.IgnoreCase);
if (regex.IsMatch(responseFromServer))
{
// background md5
List <Tuple <string, string, Task <bool> > > tasklist = new List <Tuple <string, string, Task <bool> > >();
MatchCollection matchs = regex.Matches(responseFromServer);
for (int i = 0; i < matchs.Count; i++)
{
string hash = matchs[i].Groups[1].Value.ToString();
string file = matchs[i].Groups[2].Value.ToString();
Task <bool> ismatch = Task <bool> .Factory.StartNew(() => MD5File(file, hash));
tasklist.Add(new Tuple <string, string, Task <bool> >(file, hash, ismatch));
}
// parallel download
ParallelOptions opt = new ParallelOptions()
{
MaxDegreeOfParallelism = 3
};
Parallel.ForEach(tasklist, opt, task =>
//foreach (var task in tasklist)
{
string file = task.Item1;
string hash = task.Item2;
// check if existing matchs hash
task.Item3.Wait();
bool match = task.Item3.Result;
if (!match)
{
log.Info("Newer File " + file);
// check is we have already downloaded and matchs hash
if (!MD5File(file + ".new", hash))
{
if (frmProgressReporter != null)
{
frmProgressReporter.UpdateProgressAndStatus(-1, Strings.Getting + file);
}
string subdir = Path.GetDirectoryName(file) + Path.DirectorySeparatorChar;
subdir = subdir.Replace("" + Path.DirectorySeparatorChar + Path.DirectorySeparatorChar,
"" + Path.DirectorySeparatorChar);
GetNewFile(frmProgressReporter, baseurl + subdir.Replace('\\', '/'), subdir,
Path.GetFileName(file));
// check the new downloaded file matchs hash
if (!MD5File(file + ".new", hash))
{
throw new Exception("File downloaded does not match hash: " + file);
}
}
else
{
log.Info("already got new File " + file);
}
}
else
{
log.Info("Same File " + file);
if (frmProgressReporter != null)
{
frmProgressReporter.UpdateProgressAndStatus(-1, Strings.Checking + file);
}
}
});
}
}
/// <summary>
/// One update, ran every like 1/120 seconds
/// </summary>
/// <param name="e"></param>
public void Tick(FrameEventArgs e)
{
Random r = new Random();
if (currentPoints < numberOfPoints && (running || step))
{
for (int i = 0; i < emitters.Length; i++)
{
if (emitters[i].tickCounter == 0)
{
for (int j = 0; j < emitters[i].emissionRate; j++)
{
float Xrad = (float)Math.Sqrt(r.NextDouble()) * emitters[i].Xradius;
float Zrad = (float)Math.Sqrt(r.NextDouble()) * emitters[i].Zradius;
float angle = (float)r.NextDouble() * (float)Math.PI * 2;
Vector3 pos = new Vector3(emitters[i].position.X, emitters[i].position.Y + Xrad * (float)Math.Cos(angle), emitters[i].position.Z + Zrad * (float)Math.Sin(angle));
particles[currentPoints] = (new Sphere(currentPoints, pos, emitters[i].velocity, 0.01f));
particles[currentPoints].color = new Vector3(1.0f, 0, 0);
particles[currentPoints].Mass = 1.0f;
particles[currentPoints].NetForce = new Vector3(0, 0, 0);
currentPoints++;
}
}
emitters[i].tickCounter++;
if (emitters[i].tickCounter > emitters[i].delay)
{
emitters[i].tickCounter = 0;
}
}
}
// If set to threading, split the taskforce up, but if the amount of points is too small then there's no point
if (threading && !(particles.Length < 100))
{
if (running || step)
{
int PPT = 10;
var options = new ParallelOptions()
{
MaxDegreeOfParallelism = 8
};
Parallel.For(0, currentPoints / PPT, options, i =>
{
simulator.PropertiesUpdate(i * PPT, (i + 1) * PPT);
});
Parallel.For(0, currentPoints / PPT, options, i =>
{
simulator.ForcesUpdate(i * PPT, (i + 1) * PPT);
});
simulator.MovementUpdate();
for (int i = 0; i < currentPoints; i++)
{
particles[i].Update(dt);
}
}
}
else
{
//particles[i].Update(dt);
if (running || step)
{
simulator.Update();
step = false;
}
}
}
private void PrefetchingTask_DoWork(object sender, DoWorkEventArgs e)
{
List <string> illusts = new List <string>();
List <string> originals = new List <string>();
List <string> avatars = new List <string>();
List <string> page_thumbs = new List <string>();
List <string> page_previews = new List <string>();
List <string> needUpdate = new List <string>();
try
{
var args = e.Argument is PrefetchingOpts ? e.Argument as PrefetchingOpts : new PrefetchingOpts();
if (!args.PrefetchingPreview)
{
return;
}
LastStartTime = DateTime.Now;
var pagesCount = CalcPagesThumbItems(Items);
GetPreviewItems(illusts, avatars, page_thumbs, page_previews, originals);
if (pagesCount != page_thumbs.Count + page_previews.Count)
{
e.Cancel = true; return;
}
var total = illusts.Count + avatars.Count + page_thumbs.Count + page_previews.Count;
if (total <= 0)
{
e.Cancel = true; return;
}
var count = total;
Percentage = count == 0 ? 100 : 0;
Comments = $"Calculating [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
State = TaskStatus.WaitingToRun;
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
needUpdate.AddRange(args.ReverseOrder ? illusts.Reverse <string>() : illusts);
needUpdate.AddRange(args.ReverseOrder ? avatars.Reverse <string>() : avatars);
needUpdate.AddRange(args.ReverseOrder ? page_thumbs.Reverse <string>() : page_thumbs);
needUpdate.AddRange(args.ReverseOrder ? page_previews.Reverse <string>() : page_previews);
foreach (var url in needUpdate.Where(url => !PrefetchedList.ContainsKey(url) && File.Exists(url.GetImageCacheFile())))
{
PrefetchedList.AddOrUpdate(url, true, (k, v) => true);
//if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false);
}
needUpdate = needUpdate.Where(url => !PrefetchedList.ContainsKey(url) || !PrefetchedList[url]).ToList();
count = needUpdate.Count;
Percentage = count == 0 ? 100 : (total - count) / (double)total * 100;
if (count == 0)
{
Comments = $"Done [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
State = TaskStatus.RanToCompletion;
}
else
{
Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
State = TaskStatus.Running;
}
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
this.DoEvents();
if (count == 0)
{
return;
}
var parallels = args.PrefetchingDownloadParallel;
if (args.ParallelPrefetching)
{
var opt = new ParallelOptions();
opt.MaxDegreeOfParallelism = parallels;
Parallel.ForEach(needUpdate, opt, (url, loopstate, urlIndex) =>
{
try
{
var file = url.GetImageCacheFile();
if (!string.IsNullOrEmpty(file))
{
if (File.Exists(file))
{
PrefetchedList.AddOrUpdate(url, true, (k, v) => true);
//if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false);
count = count - 1;
}
else
{
var _downReport = DownloadProgressActions.ContainsKey(url) ? DownloadProgressActions[url] : null;
file = url.DownloadCacheFile(args.Overwrite, progressAction: _downReport, cancelToken: PrefetchingTaskCancelTokenSource).GetAwaiter().GetResult();
if (!string.IsNullOrEmpty(file))
{
PrefetchedList.AddOrUpdate(url, true, (k, v) => true);
//if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false);
count = count - 1;
}
}
}
if (PrefetchingBgWorker.CancellationPending)
{
e.Cancel = true; State = TaskStatus.Canceled; loopstate.Stop();
}
Percentage = count == 0 ? 100 : (total - count) / (double)total * 100;
Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
State = TaskStatus.Running;
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke((double)this.Percentage, Comments, State);
}
this.DoEvents();
}
catch (Exception ex) { ex.ERROR("PREFETCHING"); }
finally { this.DoEvents(); Task.Delay(1).GetAwaiter().GetResult(); }
});
}
else
{
SemaphoreSlim tasks = new SemaphoreSlim(parallels, parallels);
foreach (var url in needUpdate)
{
if (PrefetchingBgWorker.CancellationPending)
{
e.Cancel = true; break;
}
if (tasks.Wait(-1, PrefetchingTaskCancelTokenSource.Token))
{
new Action(async() =>
{
try
{
var file = url.GetImageCacheFile();
if (!string.IsNullOrEmpty(file))
{
if (File.Exists(file))
{
PrefetchedList.AddOrUpdate(url, true, (k, v) => true);
//if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false);
count = count - 1;
}
else
{
var _downReport = DownloadProgressActions.ContainsKey(url) ? DownloadProgressActions[url] : null;
file = await url.DownloadCacheFile(args.Overwrite, progressAction: _downReport, cancelToken: PrefetchingTaskCancelTokenSource);
if (!string.IsNullOrEmpty(file))
{
PrefetchedList.AddOrUpdate(url, true, (k, v) => true);
//if (!PrefetchedList.TryAdd(url, true)) PrefetchedList.TryUpdate(url, true, false);
count = count - 1;
}
}
}
if (PrefetchingBgWorker.CancellationPending)
{
e.Cancel = true; State = TaskStatus.Canceled; return;
}
Percentage = count == 0 ? 100 : (total - count) / (double)total * 100;
Comments = $"Prefetching [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
State = TaskStatus.Running;
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
//await Task.Delay(10);
this.DoEvents();
}
catch (Exception ex) { ex.ERROR("PREFETCHING"); }
finally { if (tasks is SemaphoreSlim && tasks.CurrentCount <= parallels)
{
tasks.Release();
}
this.DoEvents(); await Task.Delay(1); }
}).Invoke(async: false);
}
}
this.DoEvents();
}
if (PrefetchingBgWorker.CancellationPending)
{
e.Cancel = true; State = TaskStatus.Canceled; return;
}
if (count >= 0 && total > 0)
{
Percentage = count == 0 ? 100 : (total - count) / (double)total * 100;
Comments = $"Done [ {count} / {total}, I:{illusts.Count} / A:{avatars.Count} / T:{page_thumbs.Count} / P:{page_previews.Count} ]";
//State = TaskStatus.RanToCompletion;
//if (ReportProgressSlim is Action) ReportProgressSlim.Invoke(async: false);
//else if (ReportProgress is Action<double, string, TaskStatus>) ReportProgress.Invoke(Percentage, Comments, State);
//this.DoEvents();
$"Prefetching Previews, Avatars, Thumbnails : {Environment.NewLine} {Comments}".ShowToast("INFO", tag: args.Name ?? Name ?? GetType().Name);
}
}
catch (Exception ex)
{
ex.ERROR("PREFETCHING");
Comments = $"Failed {Comments}";
State = TaskStatus.Faulted;
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
}
finally
{
try
{
GetOriginalImageSize(originals, e);
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
this.DoEvents();
illusts.Clear();
avatars.Clear();
page_thumbs.Clear();
page_previews.Clear();
originals.Clear();
needUpdate.Clear();
}
catch (Exception ex) { ex.ERROR("PREFETCHED"); }
if (CanPrefetching is SemaphoreSlim && CanPrefetching.CurrentCount < 1)
{
CanPrefetching.Release();
}
LastStartTime = DateTime.Now;
}
}
private bool GetOriginalImageSize(List <string> originals, DoWorkEventArgs e)
{
var result = false;
var setting = Application.Current.LoadSetting();
if (setting.QueryOriginalImageSize)
{
var args = e.Argument is PrefetchingOpts ? e.Argument as PrefetchingOpts : new PrefetchingOpts();
if (!args.PrefetchingPreview)
{
return(result);
}
State = TaskStatus.WaitingToRun;
var comments = Comments.Substring(0, Comments.IndexOf("]") + 1);
var count = originals.Count;
bool paralllel = args.ParallelPrefetching;
var parallels = args.PrefetchingDownloadParallel;
if (paralllel)
{
var opt = new ParallelOptions();
opt.MaxDegreeOfParallelism = parallels;
Parallel.ForEach(originals, opt, (url, loopstate, urlIndex) =>
{
try
{
var size = url.QueryImageFileSize(cancelToken: PrefetchingTaskCancelTokenSource).GetAwaiter().GetResult();
if (size > 0)
{
Comments = comments.Replace("]", $"] [ Q: {--count} / {originals.Count} ]");
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke((double)Percentage, Comments, State);
}
this.DoEvents();
}
}
catch (Exception ex) { ex.ERROR("PREFETCHING"); }
finally { this.DoEvents(); Task.Delay(1).GetAwaiter().GetResult(); }
});
}
else
{
SemaphoreSlim tasks = new SemaphoreSlim(parallels, parallels);
foreach (var url in originals)
{
if (PrefetchingBgWorker.CancellationPending)
{
e.Cancel = true; break;
}
if (tasks.Wait(-1, PrefetchingTaskCancelTokenSource.Token))
{
new Action(async() =>
{
try
{
var size = await url.QueryImageFileSize(cancelToken: PrefetchingTaskCancelTokenSource);
if (size > 0)
{
Comments = comments.Replace("]", $"] [ Q: {--count} / {originals.Count} ]");
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke((double)Percentage, Comments, State);
}
this.DoEvents();
}
}
catch (Exception ex) { ex.ERROR("PREFETCHING"); }
finally { if (tasks is SemaphoreSlim && tasks.CurrentCount <= parallels)
{
tasks.Release();
}
this.DoEvents(); await Task.Delay(1); }
}).Invoke(async: false);
}
}
}
$"Query Original Imagee File Size : {Environment.NewLine} Done [ {originals.Count} ]".ShowToast("INFO", tag: args.Name ?? Name ?? GetType().Name);
State = count <= 0 ? TaskStatus.RanToCompletion : TaskStatus.Faulted;
if (ReportProgressSlim is Action)
{
ReportProgressSlim.Invoke(async: false);
}
else if (ReportProgress is Action <double, string, TaskStatus> )
{
ReportProgress.Invoke(Percentage, Comments, State);
}
setting.SaveImageFileSizeData();
result = true;
}
return(result);
}
private void LoadSpeechRecognition() // fazer o que é preciso para o reconhecimento de voz
{
sre = new SpeechRecognitionEngine(new System.Globalization.CultureInfo("pt-BR")); // instanciando o reconhecedor passando a cultura da engine
sre.SetInputToDefaultAudioDevice(); // definindo o microfone como entrada de aúdio
commandsForQA = new List <string>();
commandsForQA.Add("o que é");
commandsForQA.Add("qual é");
commandsForQA.Add("defina");
commandsForQA.Add("definição de");
// vamos processar o AIML aqui
Choices cAIML = new Choices(AIML.GetWordsOrSentences()); // obtendo frases e palavras
// Vamos ler o arquivo dos comandos
string[] cmds = File.ReadAllText("comandos.txt", Encoding.UTF8).Split('$'); // lendo ele e dividindo em linhas
Choices cControls = new Choices();
cControls.Add("detecção de movimento");
// Alarme
Choices cAlarm = new Choices();
for (int i = 1; i <= 12; i++)
{
cAlarm.Add(i.ToString());
}
// Criação das Gramáticas, Choices
Choices cChats = new Choices(); // palavras ou frases de conversa
cChats.Add("bom dia");
cChats.Add("boa tarde");
cChats.Add("boa noite");
cChats.Add("jarvis você está ai?");
cChats.Add("ainda acordado jarvis?");
cChats.Add("alguma ideia jarvis?");
cChats.Add("obrigado jarvis");
Choices cDummes = new Choices(); // conversa mais desenrolada
cDummes.Add(DummeIn.InStartingConversation.ToArray());
cDummes.Add(DummeIn.InQuestionForDumme.ToArray());
cDummes.Add(DummeIn.InDoWork.ToArray());
cDummes.Add(DummeIn.InDummeStatus.ToArray());
cDummes.Add(DummeIn.InJarvis.ToArray());
Choices cCommands = new Choices(); // palavras ou frases que são comandos
// informações de hora e data
cCommands.Add("que horas são");
cCommands.Add("que dia é hoje");
cCommands.Add("data de hoje");
cCommands.Add("em que mês estamos");
cCommands.Add("em que ano estamos");
cCommands.Add("minimizar a janela principal");
cCommands.Add("mostrar janela principal");
// Comandos do programa
cCommands.Add("exibir lista de comandos");
// status do usuário
cCommands.Add("estou com sono");
cCommands.Add("estou indo dormir");
// sair do JARVIS
cCommands.Add("até mais jarvis");
// configurar o sintetizador
cCommands.Add("pare de falar");
// media player
cCommands.Add("media player");
cCommands.Add("selecionar arquivo para o media player");
cCommands.Add("pausar");
cCommands.Add("continuar");
cCommands.Add("parar");
cCommands.Add("fechar media player");
cCommands.Add("abrir diretório para reproduzir");
cCommands.Add("próximo");
cCommands.Add("anterior");
cCommands.Add("aumentar volume do media player");
cCommands.Add("diminuir volume do media player");
cCommands.Add("media player sem som");
cCommands.Add("media player com som");
cCommands.Add("media player em tela cheia");
cCommands.Add("que arquivo está tocando");
// informações do sistema
cCommands.Add("em quanto estar o uso do processador?");
cCommands.Add("quanta memória ram estar sendo usada?");
cCommands.Add("quanta mamória ram ainda há livre?");
cCommands.Add("quanta memória ram há no total?");
// Comandos, adicionar
cCommands.Add("adicionar novo comando");
// processos
cCommands.Add("detalhes dos processos");
// processList
cCommands.Add("lista de processos");
cCommands.Add("fechar o processo selecionado");
// jarvis
cCommands.Add("introdução ao assistente jarvis");
cCommands.Add("desligar computador");
cCommands.Add("reiniciar computador");
cCommands.Add("cancelar desligamento");
cCommands.Add("cancelar reinicialização");
// Youtube
cCommands.Add("tocar algo do youtube");
cCommands.Add("adicionar link para o youube");
cCommands.Add("previsão do tempo");
// controle de janelas
cCommands.Add("alterar de janela");
cCommands.Add("fechar janela");
// comandos de teclas
cCommands.Add("copiar texto selecionado");
cCommands.Add("colar texto selecionado");
cCommands.Add("salvar este arquivo");
cCommands.Add("selecionar tudo");
cCommands.Add("nova linha");
// Comandos do canal Código Logo
cCommands.Add("reproduza um vídeo do canal");
//Choices cNumbers = new Choices(File.ReadAllLines("n.txt")); // números
Choices cProcess = new Choices(); // lista de comandos
cProcess.Add("bloco de notas");
cProcess.Add("windows media player");
cProcess.Add("prompt de comando");
cProcess.Add("gerenciador de tarefas");
cProcess.Add("minhas pastas");
cProcess.Add("calculadora");
cProcess.Add("mapa de caracteres");
cProcess.Add("limpeza de disco");
cProcess.Add("gerenciamento de cores");
cProcess.Add("serviços de componente");
cProcess.Add("gerenciamento de computador");
cProcess.Add("definir programas padrão");
cProcess.Add("painel de controle");
cProcess.Add("otimizador de texto");
cProcess.Add("calibragem de cores");
cProcess.Add("desfragmentador de disco");
cProcess.Add("adicionar um novo dispositivo");
cProcess.Add("gerenciador de dispositivos");
cProcess.Add("discagem telefônica");
cProcess.Add("gerenciamento de disco");
Choices cCustomSites = new Choices(); // lista de comandos do usuário
Choices webSearch = new Choices();
StreamReader srWords = new StreamReader("words.txt", Encoding.UTF8);
while (srWords.Peek() >= 0)
{
try
{
webSearch.Add(srWords.ReadLine());
}
catch { }
}
srWords.Close();
// vamos processar o comandos.txt
for (int i = 0; i < cmds.Length; i++)
{
try
{
if (cmds[i].StartsWith("site#"))
{
cmds[i] = cmds[i].Replace("site#", "");
string[] temp = cmds[i].Split('#');
cCustomSites.Add(temp[0]); // adicionamos a palavra na gramática
dictCmdSites.Add(temp[0], temp[1]);
}
}
catch { }
}
// Gramática do alarme
GrammarBuilder gbAlarm = new GrammarBuilder();
gbAlarm.Append(new Choices("defina alarme", "alarme às", "despertador às"));
gbAlarm.Append(cAlarm);
gbAlarm.Append(new Choices("horas da manhã", "horas da tarde", "horas da noite"));
// GrammarsBuilders
GrammarBuilder gbChats = new GrammarBuilder(); // vamos criar um grammaBuilder para as conversas
gbChats.Append(cChats); // já foi feito
GrammarBuilder gbDumme = new GrammarBuilder(); // conversa solta
gbDumme.Append(cDummes);
GrammarBuilder gbCommands = new GrammarBuilder(); //para a lista de comandos
gbCommands.Append(cCommands); // feito
GrammarBuilder gbControls = new GrammarBuilder();
gbControls.Append(cControls);
/*
* GrammarBuilder gbCalculations = new GrammarBuilder(); // gramática que vai fazer cálculos
* gbCalculations.Append("quanto é"); // primeira parte
* gbCalculations.Append(cNumbers); // números
* gbCalculations.Append(new Choices("mais", "menos", "dividido por", "vezes", "porcento de"));
* gbCalculations.Append(cNumbers); // números novamente
* // essa gramática pode fazer por exemplo "quanto é 65 vezes 42", "quanto é 14 porcento de 500"
*/
GrammarBuilder gbProcess = new GrammarBuilder();
gbProcess.Append(new Choices("abrir", "fechar")); // comando
gbProcess.Append(cProcess); // adicionar lista de processos
GrammarBuilder gbCustomSites = new GrammarBuilder(); // sites e páginas
gbCustomSites.Append(new Choices("abrir", "iniciar", "carregar", "ir para o")); // parametros
gbCustomSites.Append(cCustomSites);
GrammarBuilder gbWebSearch = new GrammarBuilder();
gbWebSearch.Append(new Choices("pesquisar", "buscar", "procurar", "buscar por", "pesquisar vídeo de", "imagem de",
"imagens de", "pesquisar imagem de", "pesquisar por"));
gbWebSearch.Append(webSearch);
gbWebSearch.Append(new Choices("no youtube", "no google"));
GrammarBuilder gbQA = new GrammarBuilder(); // gramática para responder perguntas
gbQA.Append(new Choices(commandsForQA.ToArray()));
gbQA.Append(cAIML);
// Grammars
Grammar gQA = new Grammar(gbQA);
gQA.Name = "QA";
Grammar gChats = new Grammar(gbChats); // gramática das conversas
gChats.Name = "Chats"; // damos um nome para a gramática, pois vamos usa isso mais adiante
Grammar gDumme = new Grammar(gbDumme);
gDumme.Name = "Dumme"; // nome
Grammar gCommands = new Grammar(gbCommands); // gramática dos comandos
gCommands.Name = "Commands"; // nome
Grammar gCustomSites = new Grammar(gbCustomSites); // gramáticas dos sites
gCustomSites.Name = "Sites";
Grammar gAIML = new Grammar(new GrammarBuilder(cAIML));
gAIML.Name = "AIML";
/*
* Grammar gCalculations = new Grammar(gbCalculations);
* gCalculations.Name = "Calculations"; */
Grammar gProcess = new Grammar(gbProcess);
gProcess.Name = "Process";
// Agora vamos carregar as gramáticas
Grammar gWebSearch = new Grammar(gbWebSearch);
gWebSearch.Name = "Search";
Grammar gControls = new Grammar(gbControls);
gControls.Name = "Control";
// podemos fazer de várias maneiras, por enquanto vou fazer o seguinte
// Lista de gramáticas
List <Grammar> grammars = new List <Grammar>();
grammars.Add(gQA);
grammars.Add(gChats);
grammars.Add(gDumme);
grammars.Add(gCommands); // comandos
grammars.Add(gAIML);
grammars.Add(gWebSearch);
grammars.Add(gControls);
//grammars.Add(gCalculations);
grammars.Add(gProcess);
grammars.Add(gCustomSites);
ParallelOptions op = new ParallelOptions()
{
MaxDegreeOfParallelism = 4
};
Parallel.For(0, grammars.Count, op, i => // loop paralelo
{
sre.LoadGrammar(grammars[i]); // carregar gramática
});
speechRecognitionActived = true; // reconhecimento de voz ativo!
sre.SpeechRecognized += new EventHandler <SpeechRecognizedEventArgs>(reconhecido); // evento do reconhecimento
sre.AudioLevelUpdated += new EventHandler <AudioLevelUpdatedEventArgs>(audioElevou); // quando o aúdio é elevadosre
sre.SpeechRecognitionRejected += new EventHandler <SpeechRecognitionRejectedEventArgs>(rejeitado); // quando o reconhecimento de voz falhou
sre.SpeechDetected += new EventHandler <SpeechDetectedEventArgs>(vozDetectada); // alguma voz foi detectada
sre.LoadGrammarCompleted += new EventHandler <LoadGrammarCompletedEventArgs>(loaded); // gramática carregada
sre.RecognizeAsync(RecognizeMode.Multiple); // iniciar o reconhecimento async e múltiplo
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int folderThreadCount = -1,
int perFileThreadCount = -1,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
Cmdlet cmdletRunningRequest = null)
{
var allDirectories = new Stack <string>();
var allFailedFiles = new ConcurrentDictionary <string, string>();
var allFailedDirs = new List <string>();
var fileCount = 0;
var testFileCountChanged = 0;
var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive);
var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive);
var folderPathStartIndex = Path.GetDirectoryName(sourceFolderPath).Length;
if (folderPathStartIndex < 1)
{
// this is the scenario where the user is copying from the root of a drive
// such as C:\ or .\. In these cases, we simply indicate the "beginning" as the
// end of the root.
folderPathStartIndex = sourceFolderPath.Length;
}
allDirectories.Push(sourceFolderPath);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes to be copied: {2}. Total files to be copied: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{
PercentComplete = 0
};
UpdateProgress(progress, cmdletRunningRequest);
var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount;
var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount;
// we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.DefaultConnectionLimit =
Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads,
ServicePointManager.DefaultConnectionLimit);
ServicePointManager.Expect100Continue = false;
//TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in
while (allDirectories.Count > 0)
{
var currentDir = allDirectories.Pop();
string[] files;
try
{
files = Directory.GetFiles(currentDir);
if (recursive)
{
// Push the subdirectories onto the stack for traversal.
// This could also be done before handing the files.
foreach (var str in Directory.GetDirectories(currentDir))
{
allDirectories.Push(str);
}
}
}
catch
{
// update the list of folders that could not be accessed
// for later reporting to the user.
allFailedDirs.Add(currentDir);
continue;
}
// Execute in parallel if there are enough files in the directory.
// Otherwise, execute sequentially.Files are opened and processed
// synchronously but this could be modified to perform async I/O.
// NOTE: in order to write progress in a meaningful way, we have
// wrapped the parallel execution in a container task, which is
// then monitored from the main thread.
// TODO: enable resumability in the event that copy fails somewhere in the middle
var folderOptions = new ParallelOptions
{
CancellationToken = cmdletCancellationToken
};
if (folderThreadCount > 0)
{
folderOptions.MaxDegreeOfParallelism = folderThreadCount;
}
var task = Task.Run(
() =>
{
Parallel.ForEach(
files,
folderOptions,
() => 0,
(file, loopState, localCount) =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
var dataLakeFilePath = string.Format(
"{0}/{1}",
destinationFolderPath,
file.Substring(folderPathStartIndex).TrimStart('\\').Replace('\\', '/'));
// for each file we will either honor a force conversion
// to either binary or text, or attempt to determine
// if the file is either binary or text, with a default
// behavior of text.
isBinary = forceBinaryOrText
? isBinary
: GlobalMembers.BinaryFileExtension.Contains(
Path.GetExtension(file).ToLowerInvariant());
try
{
CopyFile(dataLakeFilePath, accountName, file, cmdletCancellationToken,
internalFileThreads, overwrite, resume, isBinary, null, progress);
}
catch (Exception e)
{
allFailedFiles.GetOrAdd(file, e.Message);
}
// note: we will always increment the count, since the file was seen and attempted
// this does not necessarily mean the file was successfully uploaded, as indicated by
// the warning messages that can be written out.
return(++localCount);
},
c => Interlocked.Add(ref fileCount, c));
}, cmdletCancellationToken);
while (!task.IsCompleted && !task.IsCanceled)
{
// if we somehow made it in here prior to the cancel, I want to issue a throw
cmdletCancellationToken.ThrowIfCancellationRequested();
// only update progress if the percentage has changed.
if ((int)Math.Ceiling((decimal)testFileCountChanged / totalFiles * 100)
< (int)Math.Ceiling((decimal)fileCount / totalFiles * 100))
{
testFileCountChanged = fileCount;
var percentComplete = (int)Math.Ceiling((decimal)fileCount / totalFiles * 100);
if (percentComplete > 100)
{
// in some cases we can get 101 percent complete using ceiling, however we want to be
// able to round up to full percentage values, instead of down.
percentComplete = 100;
}
progress.PercentComplete = percentComplete;
UpdateProgress(progress, cmdletRunningRequest);
}
// sleep for a half of a second.
TestMockSupport.Delay(500);
}
if (task.IsFaulted && !task.IsCanceled)
{
var ae = task.Exception;
if (ae != null)
{
if (cmdletRunningRequest != null)
{
cmdletRunningRequest.WriteWarning(
"The following errors were encountered during the copy:");
}
else
{
Console.WriteLine(@"The following errors were encountered during the copy:");
}
ae.Handle(
ex =>
{
if (ex is AggregateException)
{
var secondLevel = ex as AggregateException;
secondLevel.Handle(
secondEx =>
{
if (cmdletRunningRequest != null)
{
cmdletRunningRequest.WriteWarning(secondEx.ToString());
}
else
{
Console.WriteLine(secondEx);
}
return(true);
});
}
else
{
if (cmdletRunningRequest != null)
{
cmdletRunningRequest.WriteWarning(ex.ToString());
}
else
{
Console.WriteLine(ex);
}
}
return(true);
});
}
}
}
if (allFailedDirs.Count > 0 && !cmdletCancellationToken.IsCancellationRequested)
{
var errString =
"The following {0} directories could not be opened and their contents must be copied up with the single file copy command: {1}";
if (cmdletRunningRequest != null)
{
cmdletRunningRequest.WriteWarning(
string.Format(errString, allFailedDirs.Count, string.Join(",\r\n", allFailedDirs)));
}
else
{
Console.WriteLine(errString, allFailedDirs.Count, string.Join(",\r\n", allFailedDirs));
}
}
if (allFailedFiles.Count > 0 && !cmdletCancellationToken.IsCancellationRequested)
{
var errString =
"The following {0} files could not be copied and must be copied up with the single file copy command: {1}";
if (cmdletRunningRequest != null)
{
cmdletRunningRequest.WriteWarning(
string.Format(errString, allFailedFiles.Count, string.Join(",\r\n", allFailedFiles)));
}
else
{
Console.WriteLine(errString, allFailedFiles.Count, string.Join(",\r\n", allFailedFiles));
}
}
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
finally
{
ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit;
ServicePointManager.Expect100Continue = previousExpect100;
}
}
/// <summary>Renders a mandelbrot fractal.</summary>
/// <param name="position">The MandelbrotPosition representing the fractal boundaries to be rendered.</param>
/// <param name="imageWidth">The width in pixels of the image to create.</param>
/// <param name="imageHeight">The height in pixels of the image to create.</param>
/// <param name="parallelRendering">Whether to render the image in parallel.</param>
/// <returns>The rendered Bitmap.</returns>
public unsafe static Bitmap Create(MandelbrotPosition position, int imageWidth, int imageHeight, CancellationToken cancellationToken, bool parallelRendering)
{
// The maximum number of iterations to perform for each pixel. Higher number means better
// quality but also slower.
const int maxIterations = 256;
// In order to use the Bitmap ctor that accepts a stride, the stride must be divisible by four.
// We're using imageWidth as the stride, so shift it to be divisible by 4 as necessary.
if (imageWidth % 4 != 0)
{
imageWidth = (imageWidth / 4) * 4;
}
// Based on the fractal bounds, determine its upper left coordinate
double left = position.CenterX - (position.Width / 2);
double top = position.CenterY - (position.Height / 2);
// Get the factors that can be multiplied by row and col to arrive at specific x and y values
double colToXTranslation = position.Width / (double)imageWidth;
double rowToYTranslation = position.Height / (double)imageHeight;
// Create the byte array that will store the rendered color indices
int pixels = imageWidth * imageHeight;
byte[] data = new byte[pixels]; // initialized to all 0s, which equates to all black based on the default palette
// Generate the fractal using the mandelbrot formula : z = z^2 + c
// Parallel implementation
if (parallelRendering)
{
var options = new ParallelOptions {
CancellationToken = cancellationToken
};
Parallel.For(0, imageHeight, options, row =>
{
double initialY = row * rowToYTranslation + top;
fixed(byte *ptr = data)
{
byte *currentPixel = &ptr[row * imageWidth];
for (int col = 0; col < imageWidth; col++, currentPixel++)
{
Complex c = new Complex(col * colToXTranslation + left, initialY);
Complex z = c;
for (int iteration = 0; iteration < maxIterations; iteration++)
{
if (z.Magnitude > 4)
{
*currentPixel = (byte)iteration;
break;
}
z = (z * z) + c;
}
}
}
});
}
// Sequential implementation
else
{
for (int row = 0; row < imageHeight; row++)
{
cancellationToken.ThrowIfCancellationRequested();
double initialY = row * rowToYTranslation + top;
fixed(byte *ptr = data)
{
byte *currentPixel = &ptr[row * imageWidth];
for (int col = 0; col < imageWidth; col++, currentPixel++)
{
Complex c = new Complex(col * colToXTranslation + left, initialY);
Complex z = c;
for (int iteration = 0; iteration < maxIterations; iteration++)
{
if (z.Magnitude > 4)
{
*currentPixel = (byte)iteration;
break;
}
z = (z * z) + c;
}
}
}
}
;
}
// Produce a Bitmap from the byte array of color indices and return it
fixed(byte *ptr = data)
{
using (Bitmap tempBitmap = new Bitmap(imageWidth, imageHeight, imageWidth, PixelFormat.Format8bppIndexed, (IntPtr)ptr))
{
Bitmap bitmap = tempBitmap.Clone(new Rectangle(0, 0, tempBitmap.Width, tempBitmap.Height), PixelFormat.Format8bppIndexed);
UpdatePalette(bitmap);
return(bitmap);
}
}
}
public static void WriteTypes(Dictionary <string, ParsedType> types, string outputDirectory)
{
var di = new System.IO.DirectoryInfo(outputDirectory);
string apibase = di.Name;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = 100
};
Parallel.ForEach(types, parallelOptions, (keyValue) =>
{
ParsedType basetype = keyValue.Value;
if (!basetype.IsPublic)
{
return;
}
var content = new StringBuilder();
content.AppendLine("---");
content.AppendLine($"title: \"{basetype.Name}\"");
content.AppendLine($"date: {DateTime.Now.ToString("u")}");
content.AppendLine("draft: false");
content.AppendLine("---");
content.AppendLine();
content.AppendLine($"*Namespace: [{basetype.Namespace}](../)*");
content.AppendLine();
string baseTypeSummary = basetype.Summary();
if (!string.IsNullOrEmpty(baseTypeSummary))
{
content.AppendLine(baseTypeSummary);
}
if (basetype.IsClass)
{
content.AppendLine("```cs");
string[] attributes = basetype.GetAttributes();
for (int i = 0; i < attributes.Length; i++)
{
content.AppendLine(attributes[i]);
}
content.Append($"public class {basetype.Name}");
string[] baseList = basetype.GetBaseList(null);
if (baseList != null)
{
for (int i = 0; i < baseList.Length; i++)
{
if (i == 0)
{
content.Append($": {baseList[i]}");
}
else
{
content.Append($", {baseList[i]}");
}
}
}
content.AppendLine();
content.AppendLine("```");
}
if (basetype.Members == null)
{
return; // TODO: Figure out this case
}
ParsedMemberType state = ParsedMemberType.None;
foreach (var item in basetype.Members)
{
if (item.IsEvent && state != ParsedMemberType.Event)
{
content.AppendLine("## Events");
state = ParsedMemberType.Event;
}
if (item.IsProperty && state != ParsedMemberType.Property)
{
content.AppendLine("## Properties");
state = ParsedMemberType.Property;
}
if (item.IsMethod && state != ParsedMemberType.Method)
{
content.AppendLine("## Methods");
state = ParsedMemberType.Method;
}
if (item.IsConstructor && state != ParsedMemberType.Constructor)
{
content.AppendLine("## Constructors");
state = ParsedMemberType.Constructor;
}
content.AppendLine();
string signature = item.Signature(false);
var returntype = item.ReturnType(types.Values);
if (returntype != null && returntype != item.ParentType)
{
string rn = returntype.Name;
int index = signature.IndexOf(rn);
string link = ParsedTypePath(apibase, returntype);
string s = "";
if (index > 0)
{
s = signature.Substring(0, index);
}
s += $"[{signature.Substring(index, rn.Length)}]({link}){signature.Substring(index + rn.Length)}";
signature = s;
}
content.AppendLine(signature);
content.AppendLine($": {item.Summary()}");
string returnString = item.ReturnDocString();
if (!string.IsNullOrWhiteSpace(returnString))
{
content.AppendLine($": Returns - {returnString}");
}
if (!item.Since.Equals("5.0")) // no need to add since tags initial items
{
content.AppendLine($": since {item.Since}");
}
}
string name = basetype.Name;
string directory = OutputDirectoryFromNamespace(outputDirectory, basetype.Namespace);
string path = System.IO.Path.Combine(directory, name.ToLower() + ".md");
if (WriteContent(content, path, true))
{
Console.WriteLine($"(write) {name}");
}
else
{
Console.WriteLine($"(no change) {name}");
}
});
}
