public IEnumerator <KeyValuePair <T, V> > GetEnumerator() { Cleanup(); return(Memory .AsEnumerable() .Select(p => new KeyValuePair <T, V>(p.Key, p.Value.Key)) .GetEnumerator()); }
internal IEnumerable <KeyValuePair <string, string> > AsKafkaConfig() { if (_kafkaConfig == null) { if (string.IsNullOrWhiteSpace(Servers)) { throw new ArgumentNullException(nameof(Servers)); } MainConfig["debug"] = "consumer, broker"; MainConfig["bootstrap.servers"] = Servers; MainConfig["queue.buffering.max.ms"] = "10"; MainConfig["enable.auto.commit"] = "false"; MainConfig["log.connection.close"] = "true"; MainConfig["request.timeout.ms"] = "3000"; MainConfig["message.timeout.ms"] = "5000"; MainConfig["reconnect.backoff.ms"] = "5000"; MainConfig["retry.backoff.ms"] = "50"; MainConfig["security.protocol"] = SecurityProtocol.Protocol; if (!string.IsNullOrEmpty(SSLCertificatePath)) { if (!File.Exists(SSLCertificatePath)) { throw new FileNotFoundException(SSLCertificatePath); } MainConfig["ssl.ca.location"] = SSLCertificatePath; } _kafkaConfig = MainConfig.AsEnumerable(); } return(_kafkaConfig); }
public IEnumerable <KeyValuePair <string, object> > AsKafkaSetting() { if (_kafkaSetting == null) { if (string.IsNullOrWhiteSpace(Servers)) { throw new ArgumentNullException(nameof(Servers)); } _setting["bootstrap.servers"] = Servers; _setting["queue.buffering.max.ms"] = "10"; _setting["socket.blocking.max.ms"] = "10"; _setting["enable.auto.commit"] = "false"; _setting["log.connection.close"] = "false"; _setting["group.id"] = "sample.app.NetCore"; //_setting["group.id"] = new Guid().ToString(); //ProduceAsync是按照顺序发送的如果发生异常 比如网络不可达,然后kafka client会重试, //如果要按照顺序重新发送则设置下面的配置,但是会降低kafka的吞吐量。 //_setting["queue.buffering.max.ms"] = "1"; //_setting["socket.blocking.max.ms"] = "1"; //如果不想批处理则设置如下 //_setting["default.topic.config"] = new Dictionary<string, object> //{ // ["acks"] = 1 //}; _kafkaSetting = _setting.AsEnumerable(); } return(_kafkaSetting); }
public void ShowStatus() { Console.Clear(); Console.WriteLine("=============================="); foreach (var job in _jobs.AsEnumerable()) { Console.WriteLine($"{job.Key} has instances: {job.Value}"); } Console.WriteLine("=============================="); }
public List <Player> GetWaitingPlayers() { List <Player> players = new List <Player>(); foreach (var element in waitingForGame.AsEnumerable()) { players.Add(element.Value); } return(players); }
private void Compile ( string[] referencedAssemblies , string sourceCode ) { CodeDomProvider codeDomProvider = CodeDomProvider.CreateProvider("CSharp"); var compilerParameters = new CompilerParameters(); Array .ForEach ( referencedAssemblies , (x) => { compilerParameters .ReferencedAssemblies .Add(x); } ); compilerParameters.GenerateExecutable = false; compilerParameters.GenerateInMemory = true; //Console.WriteLine(code); var compilerResults = codeDomProvider .CompileAssemblyFromSource ( compilerParameters , sourceCode ); var assembly = compilerResults.CompiledAssembly; var codes = _sourceCodes.AsEnumerable(); var stringBuilder = new StringBuilder(); foreach (var kvp in codes) { MethodInfo mi = assembly .GetType("Microshaoft.Temp.InvokersManager") .GetMethod(kvp.Key); Delegate invoker = (Delegate)mi.Invoke(null, null); kvp.Value.MethodInvoker = invoker; } }
/// <inheritdoc /> public Task <IEnumerable <PersistedGrant> > GetAllAsync(PersistedGrantFilter filter) { var query = _repository.AsEnumerable(); if (!string.IsNullOrEmpty(filter.ClientId)) { query = query.Where(x => x.Value.ClientId == filter.ClientId); } if (!string.IsNullOrEmpty(filter.SubjectId)) { query = query.Where(x => x.Value.SubjectId == filter.SubjectId); } if (!string.IsNullOrEmpty(filter.SessionId)) { query = query.Where(x => x.Value.SessionId == filter.SessionId); } var items = query.Select(x => x.Value).AsEnumerable(); return(Task.FromResult(items)); }
private static IEnumerable <KeyValuePair <string, string> > DiscoverUnitTestProjectsAndAssemblyNames(string projectDirectory) { var csProjectFiles = Directory.EnumerateFiles(projectDirectory, "*.csproj", SearchOption.AllDirectories); var projectFileToAssemblyMap = new ConcurrentDictionary <string, string>(); Parallel.ForEach(csProjectFiles, csProjectFile => { var(isUnitTestProject, assemblyName) = DetermineIfProjectIsUnitTestProject(csProjectFile); if (isUnitTestProject) { projectFileToAssemblyMap.TryAdd(csProjectFile, assemblyName); } }); return(projectFileToAssemblyMap.AsEnumerable()); }
internal IEnumerable <KeyValuePair <string, object> > AsKafkaConfig() { if (_kafkaConfig == null) { if (string.IsNullOrWhiteSpace(Servers)) { throw new ArgumentNullException(nameof(Servers)); } MainConfig["bootstrap.servers"] = Servers; MainConfig["queue.buffering.max.ms"] = "10"; MainConfig["socket.blocking.max.ms"] = "10"; MainConfig["enable.auto.commit"] = "false"; MainConfig["log.connection.close"] = "false"; _kafkaConfig = MainConfig.AsEnumerable(); } return(_kafkaConfig); }
internal IEnumerable <KeyValuePair <string, string> > AsKafkaConfig() { if (_kafkaConfig == null) { if (string.IsNullOrWhiteSpace(Servers)) { throw new ArgumentNullException(nameof(Servers)); } MainConfig["bootstrap.servers"] = Servers; MainConfig["queue.buffering.max.ms"] = "10"; MainConfig["allow.auto.create.topics"] = "true"; MainConfig["enable.auto.commit"] = "false"; MainConfig["log.connection.close"] = "false"; MainConfig["request.timeout.ms"] = "3000"; MainConfig["message.timeout.ms"] = "5000"; _kafkaConfig = MainConfig.AsEnumerable(); } return(_kafkaConfig); }
/// <inheritdoc cref="ICollection{TValue}.CopyTo"/> public void CopyTo(KeyValuePair <TKey, TValue>[] array, int arrayIndex) { var keyValuePairs = _Dictionary.AsEnumerable().ToArray(); keyValuePairs.CopyTo(array, arrayIndex); }
private void GetApps2(int pages = -1, int apps = -1, Operation operation = Operation.Full) { Store.Conventions.MaxNumberOfRequestsPerSession = 820; const string sitemapIndex = "http://apps.microsoft.com/windows/sitemap_index.xml"; var prefixLength = "http://apps.microsoft.com/windows/".Length; XNamespace xmlns = XNamespace.Get("http://www.sitemaps.org/schemas/sitemap/0.9"); XmlNameTable table = new NameTable(); var manager = new XmlNamespaceManager(table); manager.AddNamespace("", xmlns.NamespaceName); IEnumerable <string> sitemapPages; using (var response = _httpClient.GetAsync(sitemapIndex, HttpCompletionOption.ResponseContentRead).Result) { if (response.StatusCode != HttpStatusCode.OK) { return; } using (var stream = response.Content.ReadAsStreamAsync().Result) { XDocument sitemap = XDocument.Load(stream); var sitemapIndexElem = sitemap.Element(xmlns.GetName("sitemapindex")); var sitemaps = sitemapIndexElem.Elements(xmlns.GetName("sitemap")); sitemapPages = sitemaps .Select(x => x.Element(xmlns.GetName("loc"))).Select(x => x.Value); } } var globalAppsToCultures = new ConcurrentDictionary <string, AppInfo2>(); if (pages > 0) { sitemapPages = sitemapPages.Take(pages); } Parallel.ForEach( sitemapPages, new ParallelOptions { MaxDegreeOfParallelism = 2 }, sitemapPage => { Console.WriteLine("Requesting: {0}", sitemapPage); using (var response = _httpClient.GetAsync(sitemapPage, HttpCompletionOption.ResponseContentRead).Result) { if (response.StatusCode != HttpStatusCode.OK) { return; } Console.WriteLine("Received: {0}", sitemapPage); using (var stream = response.Content.ReadAsStreamAsync().Result) { Console.WriteLine("Loading: {0}", sitemapPage); XDocument sitemapPageContent = XDocument.Load(stream); var urlsetElem = sitemapPageContent.Element(xmlns.GetName("urlset")); var urlsElems = urlsetElem.Elements(xmlns.GetName("url")); var appsLinks = urlsElems.Select(x => x.Element(xmlns.GetName("loc"))) .Select(x => x.Value); foreach (var appLink in appsLinks) { var appId = appLink.Substring(appLink.LastIndexOf('/') + 1); var culture = appLink.Substring(prefixLength, appLink.IndexOf('/', prefixLength) - prefixLength); if (appId == "ROW") { continue; } globalAppsToCultures.AddOrUpdate( appId, new AppInfo2 { Id = appId, Url = appLink, Countries = new List <string>(new[] { culture }) }, (c, ai) => { ai.Countries.Add(culture); return(ai); } ); } Console.WriteLine("Total Found {0} applications", globalAppsToCultures.Count); } } }); AppIds = new HashSet <string>(globalAppsToCultures.Keys); if (LoadIdsOnly) { return; } var toProcess = globalAppsToCultures.AsEnumerable(); if (apps > 0) { toProcess = toProcess.Take(apps); } var toProcessList = toProcess.ToList(); var countdown = new CountdownEvent(toProcessList.Count); var commands = new List <ICommandData>(); Parallel.ForEach( toProcessList, new ParallelOptions { MaxDegreeOfParallelism = 8 }, (kvp, pls, index) => { using (var session = Store.OpenSession()) { try { Console.WriteLine("({0}/{1} - {2} remaining) Loading app {3}", index, globalAppsToCultures.Count, countdown.CurrentCount, kvp.Value.Id); var existing = session.Load <App>("apps/" + kvp.Value.Id); try { var app = GetAppDetails2(kvp.Value.Id, kvp.Value.Countries); if (app == null) { return; } if (existing != null) { if ((app.Price != null && existing.Price != null && app.Price.Amount == existing.Price.Amount) || app.LastUpdated == existing.LastUpdated) { return; } app.FriendlyUrl = existing.FriendlyUrl; app.PreviousPrice = existing.Price; } var putCommandData = new PutCommandData { Key = app.Id, Document = RavenJObject.FromObject(app), Metadata = new RavenJObject { { "Raven-Entity-Name", new RavenJValue("Apps") }, { "Raven-Clr-Type", new RavenJValue("Win8Apps.Model.App, Win8Apps.Model") } } }; lock (commands) { commands.Add(putCommandData); } } catch (Exception ex) { } } finally { countdown.Signal(); } } }); countdown.Wait(); if (commands.Count > 0) { const int batchSize = 512; int handled = 0; do { var commandsToSave = commands .Skip(handled) .Take(batchSize) .ToList(); Store.DatabaseCommands.Batch(commandsToSave); handled += commandsToSave.Count(); Console.WriteLine("Saving {0} {1}/{2} applications", commandsToSave.Count(), handled, commands.Count); Thread.Sleep(5000); } while (handled < commands.Count); } }
// TODO: Move above into DetectedProc class methods /// <summary> /// Group address spaces into related buckets /// /// We will assign an address space ID to each detected proc so we know what process belongs with who /// After AS grouping we will know what EPTP belongs to which AS since one of the DP's will have it's CR3 in the VMCS /// /// Yes it's a bit complicated. /// /// The overall procedure however is straight forward in that; /// /// * For every detected process /// Bucket into groups which are the "Address spaces" that initially are /// /// (a) based on kernel address space similarities /// and then /// (b) based on what VMCS value was found pointing to that group /// /// This ensures that if we have several hypervisors with a possibly identical kernel grouping (i.e. the PFN's /// were used by each kernel were identical), they are disambiguated by the VMCS. (Which can be validated later) /// /// The benefit here is that brute forcing at this stage is fairly expensive and can lead to significant overhead, there does /// tend to be some outliers for large systems that need to be looked at more to determine who they belong too. Nevertheless, it's /// inconsequential if they are grouped with the appropriate AS since even if they are isolated into their own 'AS' this is an artificial /// construct for our book keeping. The net result is that even if some process is grouped by itself due to some aggressive variation in /// kernel PFN' use (lots of dual mapped memory/MDL's or something), it's still able to be dumped and analyzed. /// /// /// </summary> /// <param name="pTypes">Types to scan for, this is of the already detected processes list so it's already filtered really</param> public void GroupAS(PTType pTypes = PTType.UNCONFIGURED) { var PT2Scan = pTypes == PTType.UNCONFIGURED ? PTType.ALL : pTypes; //if (Phase >=3 && OverRidePhase) // return; // To join an AS group we want to see > 50% correlation which is a lot considering were only interoperating roughly 10-20 values (more like 12) var p = from proc in Processes where (((proc.PageTableType & PT2Scan) == proc.PageTableType)) orderby proc.CR3Value ascending select proc; ASGroups = new ConcurrentDictionary<int, ConcurrentBag<DetectedProc>>(); // we trim out the known recursive/self entries since they will naturally not be equivalent var AlikelyKernelSet = from ptes in p.First().TopPageTablePage where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key) select ptes.Value; int totUngrouped = Processes.Count(); int CurrASID = 1; int LastGroupTotal = 0; var grouped = new ConcurrentBag<DetectedProc>(); WriteLine($"Scanning for group correlations total processes {totUngrouped}"); ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>(); while (true) { ForegroundColor = ConsoleColor.Yellow; Parallel.ForEach(p, (proc) => { var currKern = from ptes in proc.TopPageTablePage where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key) select ptes.Value; var interSection = currKern.Intersect(AlikelyKernelSet); var correlated = interSection.Count() * 1.00 / AlikelyKernelSet.Count(); // add this detected CR3/process address space to an address space grouping when // the kernel range is above the acceptable threshold, the group does not contain this proc // and this proc is not already joined into another group if (correlated > GroupThreshold && !ASGroups[CurrASID].Contains(proc) && proc.AddressSpaceID == 0) { WriteColor(ConsoleColor.Cyan, $"MemberProces: Group {CurrASID} Type [{proc.PageTableType}] GroupCorrelation [{correlated:P3}] PID [{proc.CR3Value:X}]"); proc.AddressSpaceID = CurrASID; ASGroups[CurrASID].Add(proc); // global list to quickly scan grouped.Add(proc); } }); ForegroundColor = ConsoleColor.Yellow; var totGrouped = (from g in ASGroups.Values select g).Sum(x => x.Count()); WriteLine($"Finished Group {CurrASID} collected size {ASGroups[CurrASID].Count()}, continuing to group"); // if there is more work todo, setup an entry for testing if (totGrouped < totUngrouped) { // if we wind up here // there has been no forward progress in isolating further groups if(LastGroupTotal == totGrouped) { ForegroundColor = ConsoleColor.Red; WriteLine($"Terminating with non-grouped process candidates. GroupThreshold may be too high. {GroupThreshold}"); var pz = from px in Processes where px.AddressSpaceID == 0 select px; // just add the ungrouped processes as a single each bare metal // unless it has an existing VMCS pointer foreach (var px in pz) { WriteLine(px); CurrASID++; px.AddressSpaceID = CurrASID; ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>() { px }; var isCandidate = from pvmcs in scan.HVLayer where pvmcs.gCR3 == px.CR3Value select pvmcs; if (isCandidate.Count() > 0) { px.CandidateList = new List<VMCS>(isCandidate); px.vmcs = px.CandidateList.First(); WriteColor( ConsoleColor.White, $"Detected ungrouped {px.CR3Value} as a candidate under {px.CandidateList.Count()} values (first){px.vmcs.EPTP}"); } } ForegroundColor = ConsoleColor.Yellow; break; } CurrASID++; ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>(); WriteLine($"grouped count ({totGrouped}) is less than total process count ({totUngrouped}, rescanning...)"); LastGroupTotal = totGrouped; } else break; // we grouped them all! /// Isolate next un-grouped PageTable var UnGroupedProc = from nextProc in Processes where !grouped.Contains(nextProc) select nextProc; AlikelyKernelSet = from ptes in UnGroupedProc.First().TopPageTablePage where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key) select ptes.Value; } Console.WriteLine($"Done All process groups."); // after grouping link VMCS back to the group who 'discovered' the VMCS in the first place! var eptpz = VMCSs.Values.GroupBy(eptz => eptz.EPTP).OrderBy(eptx => eptx.Key).Select(ept => ept.First()).ToArray(); // find groups dominated by each vmcs var VMCSGroup = from aspace in ASGroups.AsEnumerable() from ept in eptpz where aspace.Value.Any(adpSpace => adpSpace == ept.dp) select new { AS = aspace, EPTctx = ept }; // link the proc back into the eptp foreach (var ctx in VMCSGroup) foreach (var dp in ctx.AS.Value) { if(dp.CandidateList == null) dp.CandidateList = new List<VMCS>(); dp.vmcs = ctx.EPTctx; dp.CandidateList.Add(ctx.EPTctx); } // resort by CR3 foreach (var ctx in ASGroups.Values) { var dpz = from d in ctx orderby d.CR3Value descending select d; if (dpz.Count() >= 1) { var aspace = dpz.First().AddressSpaceID; ASGroups[aspace] = new ConcurrentBag<DetectedProc>(dpz); } } Phase = 4; // were good, all Processes should have a VMCS if applicable and be identifiable by AS ID }
public IEnumerable <KeyValuePair <string, ITimedTask> > GetAllTasks() { return(tasks.AsEnumerable()); }
/// <summary> /// returns filtered and ordered list of games /// </summary> /// <param name="titleFilter"></param> /// <param name="filter"></param> /// <param name="order"></param> /// <param name="ascending"></param> /// <returns></returns> public static string[] ListGames(string filterCommand, GameFilter filter, GameOrder order, bool ascending) { var games = Games.AsEnumerable(); if (!string.IsNullOrEmpty(filterCommand)) { try { var commands = filterCommand.Trim().ToLower().Split(new char[] { '&' }, StringSplitOptions.RemoveEmptyEntries); foreach (var command in commands) { if (command.StartsWith("*")) { // remove all white spaces in string var c = Regex.Replace(command, @"\s+", string.Empty); var op = GetOperatorFromCommand(c); // DATE filters if (c.Contains("*beaten")) { if (GetDateFromCommand(c.Replace($"*beaten{ op }", string.Empty), out int?year, out int?month, out int?day)) { games = games.Where(g => EvaluateDates(g.Value.Completed, op, year, month, day)); Debug.WriteLine($"beaten{ op }{ year }/{ month }/{ day }"); } } else if (c.Contains("*added")) { if (GetDateFromCommand(c.Replace($"*added{ op }", string.Empty), out int?year, out int?month, out int?day)) { games = games.Where(g => EvaluateDates(g.Value.Added, op, year, month, day)); Debug.WriteLine($"added{ op }{ year }/{ month }/{ day }"); } } else if (c.Contains("*played")) { if (GetDateFromCommand(c.Replace($"*played{ op }", string.Empty), out int?year, out int?month, out int?day)) { games = games.Where(g => EvaluateDates(g.Value.DateTimesPlayed, op, year, month, day)); Debug.WriteLine($"played{ op }{ year }/{ month }/{ day }"); } } // NUMBER filters else if (c.Contains("*playcount")) { if (GetNumberFromCommand(c.Replace($"*playcount{ op }", string.Empty), out int?count)) { games = games.Where(g => EvaluateNum(g.Value.PlayCount, op, count.Value)); Debug.WriteLine($"playcount{ op }{ count }"); } } else if (c.Contains("*rating")) { if (GetNumberFromCommand(c.Replace($"*rating{ op }", string.Empty), out int?rating)) { games = games.Where(g => EvaluateNum(g.Value.Rating, op, rating.Value)); Debug.WriteLine($"rating{ op }{ rating }"); } } } else { // filter by game title games = games.Where(g => g.Value.Title.ToLower().Contains(command)); } } } catch (Exception ex) { Log.WriteLine(ex.ToString()); games = Games.AsEnumerable(); } } switch (filter) { case GameFilter.All: break; case GameFilter.Installed: games = games.Where(g => !g.Value.Removed); break; case GameFilter.Removed: games = games.Where(g => g.Value.Removed); break; case GameFilter.Beaten: games = games.Where(g => g.Value.Completed.HasValue); break; case GameFilter.Unbeaten: games = games.Where(g => !g.Value.Completed.HasValue && !g.Value.Removed); break; case GameFilter.WithControllerSupport: games = games.Where(g => g.Value.GamepadFriendly == true && !g.Value.Removed); break; case GameFilter.Unidentified: games = games.Where(g => g.Value.Image == null); break; default: throw new NotImplementedException(filter.ToString()); } switch (order) { case GameOrder.Title: games = games.OrderBy(g => g.Value.Title); break; case GameOrder.PlayTime: games = games.OrderBy(g => g.Value.TotalTimePlayed); break; case GameOrder.PlayCount: games = games.OrderBy(g => g.Value.PlayCount); break; case GameOrder.Rating: games = games.OrderBy(g => g.Value.Rating); break; case GameOrder.BeatTime: games = games.OrderBy(g => g.Value.BeatenIn); break; case GameOrder.DateAdded: games = games.OrderBy(g => g.Value.Added); break; case GameOrder.LastPlayed: { games = games .Where(g => !g.Value.LastPlayed.HasValue) .OrderByDescending(g => g.Value.Added) .Concat(games .Where(g => g.Value.LastPlayed.HasValue) .OrderBy(g => g.Value.LastPlayed)); } break; default: throw new NotImplementedException(order.ToString()); } var result = games.Select(g => g.Key); if (!ascending) { result = result.Reverse(); } return(result.ToArray()); }
private static IReadOnlyDictionary <WittyerType, IReadOnlyList <TResult> > ToReadOnly <TSource, TResult>( ConcurrentDictionary <WittyerType, List <TSource> > variants) where TSource : TResult => new ReadOnlyDictionary <WittyerType, IReadOnlyList <TResult> >(variants.AsEnumerable() .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Cast <TResult>().ToReadOnlyList()));