public IEnumerator <KeyValuePair <T, V> > GetEnumerator()
{
Cleanup();
return(Memory
.AsEnumerable()
.Select(p => new KeyValuePair <T, V>(p.Key, p.Value.Key))
.GetEnumerator());
}
internal IEnumerable <KeyValuePair <string, string> > AsKafkaConfig()
{
if (_kafkaConfig == null)
{
if (string.IsNullOrWhiteSpace(Servers))
{
throw new ArgumentNullException(nameof(Servers));
}
MainConfig["debug"] = "consumer, broker";
MainConfig["bootstrap.servers"] = Servers;
MainConfig["queue.buffering.max.ms"] = "10";
MainConfig["enable.auto.commit"] = "false";
MainConfig["log.connection.close"] = "true";
MainConfig["request.timeout.ms"] = "3000";
MainConfig["message.timeout.ms"] = "5000";
MainConfig["reconnect.backoff.ms"] = "5000";
MainConfig["retry.backoff.ms"] = "50";
MainConfig["security.protocol"] = SecurityProtocol.Protocol;
if (!string.IsNullOrEmpty(SSLCertificatePath))
{
if (!File.Exists(SSLCertificatePath))
{
throw new FileNotFoundException(SSLCertificatePath);
}
MainConfig["ssl.ca.location"] = SSLCertificatePath;
}
_kafkaConfig = MainConfig.AsEnumerable();
}
return(_kafkaConfig);
}
public IEnumerable <KeyValuePair <string, object> > AsKafkaSetting()
{
if (_kafkaSetting == null)
{
if (string.IsNullOrWhiteSpace(Servers))
{
throw new ArgumentNullException(nameof(Servers));
}
_setting["bootstrap.servers"] = Servers;
_setting["queue.buffering.max.ms"] = "10";
_setting["socket.blocking.max.ms"] = "10";
_setting["enable.auto.commit"] = "false";
_setting["log.connection.close"] = "false";
_setting["group.id"] = "sample.app.NetCore";
//_setting["group.id"] = new Guid().ToString();
//ProduceAsync是按照顺序发送的如果发生异常 比如网络不可达，然后kafka client会重试，
//如果要按照顺序重新发送则设置下面的配置，但是会降低kafka的吞吐量。
//_setting["queue.buffering.max.ms"] = "1";
//_setting["socket.blocking.max.ms"] = "1";
//如果不想批处理则设置如下
//_setting["default.topic.config"] = new Dictionary<string, object>
//{
// ["acks"] = 1
//};
_kafkaSetting = _setting.AsEnumerable();
}
return(_kafkaSetting);
}
public void ShowStatus()
{
Console.Clear();
Console.WriteLine("==============================");
foreach (var job in _jobs.AsEnumerable())
{
Console.WriteLine($"{job.Key} has instances: {job.Value}");
}
Console.WriteLine("==============================");
}
public List <Player> GetWaitingPlayers()
{
List <Player> players = new List <Player>();
foreach (var element in waitingForGame.AsEnumerable())
{
players.Add(element.Value);
}
return(players);
}
private void Compile
(
string[] referencedAssemblies
, string sourceCode
)
{
CodeDomProvider codeDomProvider = CodeDomProvider.CreateProvider("CSharp");
var compilerParameters = new CompilerParameters();
Array
.ForEach
(
referencedAssemblies
, (x) =>
{
compilerParameters
.ReferencedAssemblies
.Add(x);
}
);
compilerParameters.GenerateExecutable = false;
compilerParameters.GenerateInMemory = true;
//Console.WriteLine(code);
var compilerResults = codeDomProvider
.CompileAssemblyFromSource
(
compilerParameters
, sourceCode
);
var assembly = compilerResults.CompiledAssembly;
var codes = _sourceCodes.AsEnumerable();
var stringBuilder = new StringBuilder();
foreach (var kvp in codes)
{
MethodInfo mi = assembly
.GetType("Microshaoft.Temp.InvokersManager")
.GetMethod(kvp.Key);
Delegate invoker = (Delegate)mi.Invoke(null, null);
kvp.Value.MethodInvoker = invoker;
}
}
/// <inheritdoc />
public Task <IEnumerable <PersistedGrant> > GetAllAsync(PersistedGrantFilter filter)
{
var query = _repository.AsEnumerable();
if (!string.IsNullOrEmpty(filter.ClientId))
{
query = query.Where(x => x.Value.ClientId == filter.ClientId);
}
if (!string.IsNullOrEmpty(filter.SubjectId))
{
query = query.Where(x => x.Value.SubjectId == filter.SubjectId);
}
if (!string.IsNullOrEmpty(filter.SessionId))
{
query = query.Where(x => x.Value.SessionId == filter.SessionId);
}
var items = query.Select(x => x.Value).AsEnumerable();
return(Task.FromResult(items));
}
private static IEnumerable <KeyValuePair <string, string> > DiscoverUnitTestProjectsAndAssemblyNames(string projectDirectory)
{
var csProjectFiles = Directory.EnumerateFiles(projectDirectory, "*.csproj", SearchOption.AllDirectories);
var projectFileToAssemblyMap = new ConcurrentDictionary <string, string>();
Parallel.ForEach(csProjectFiles, csProjectFile =>
{
var(isUnitTestProject, assemblyName) = DetermineIfProjectIsUnitTestProject(csProjectFile);
if (isUnitTestProject)
{
projectFileToAssemblyMap.TryAdd(csProjectFile, assemblyName);
}
});
return(projectFileToAssemblyMap.AsEnumerable());
}
internal IEnumerable <KeyValuePair <string, object> > AsKafkaConfig()
{
if (_kafkaConfig == null)
{
if (string.IsNullOrWhiteSpace(Servers))
{
throw new ArgumentNullException(nameof(Servers));
}
MainConfig["bootstrap.servers"] = Servers;
MainConfig["queue.buffering.max.ms"] = "10";
MainConfig["socket.blocking.max.ms"] = "10";
MainConfig["enable.auto.commit"] = "false";
MainConfig["log.connection.close"] = "false";
_kafkaConfig = MainConfig.AsEnumerable();
}
return(_kafkaConfig);
}
internal IEnumerable <KeyValuePair <string, string> > AsKafkaConfig()
{
if (_kafkaConfig == null)
{
if (string.IsNullOrWhiteSpace(Servers))
{
throw new ArgumentNullException(nameof(Servers));
}
MainConfig["bootstrap.servers"] = Servers;
MainConfig["queue.buffering.max.ms"] = "10";
MainConfig["allow.auto.create.topics"] = "true";
MainConfig["enable.auto.commit"] = "false";
MainConfig["log.connection.close"] = "false";
MainConfig["request.timeout.ms"] = "3000";
MainConfig["message.timeout.ms"] = "5000";
_kafkaConfig = MainConfig.AsEnumerable();
}
return(_kafkaConfig);
}
/// <inheritdoc cref="ICollection{TValue}.CopyTo"/>
public void CopyTo(KeyValuePair <TKey, TValue>[] array, int arrayIndex)
{
var keyValuePairs = _Dictionary.AsEnumerable().ToArray();
keyValuePairs.CopyTo(array, arrayIndex);
}
private void GetApps2(int pages = -1, int apps = -1, Operation operation = Operation.Full)
{
Store.Conventions.MaxNumberOfRequestsPerSession = 820;
const string sitemapIndex = "http://apps.microsoft.com/windows/sitemap_index.xml";
var prefixLength = "http://apps.microsoft.com/windows/".Length;
XNamespace xmlns = XNamespace.Get("http://www.sitemaps.org/schemas/sitemap/0.9");
XmlNameTable table = new NameTable();
var manager = new XmlNamespaceManager(table);
manager.AddNamespace("", xmlns.NamespaceName);
IEnumerable <string> sitemapPages;
using (var response = _httpClient.GetAsync(sitemapIndex, HttpCompletionOption.ResponseContentRead).Result)
{
if (response.StatusCode != HttpStatusCode.OK)
{
return;
}
using (var stream = response.Content.ReadAsStreamAsync().Result)
{
XDocument sitemap = XDocument.Load(stream);
var sitemapIndexElem = sitemap.Element(xmlns.GetName("sitemapindex"));
var sitemaps = sitemapIndexElem.Elements(xmlns.GetName("sitemap"));
sitemapPages = sitemaps
.Select(x => x.Element(xmlns.GetName("loc"))).Select(x => x.Value);
}
}
var globalAppsToCultures = new ConcurrentDictionary <string, AppInfo2>();
if (pages > 0)
{
sitemapPages = sitemapPages.Take(pages);
}
Parallel.ForEach(
sitemapPages,
new ParallelOptions {
MaxDegreeOfParallelism = 2
},
sitemapPage =>
{
Console.WriteLine("Requesting: {0}", sitemapPage);
using (var response = _httpClient.GetAsync(sitemapPage, HttpCompletionOption.ResponseContentRead).Result)
{
if (response.StatusCode != HttpStatusCode.OK)
{
return;
}
Console.WriteLine("Received: {0}", sitemapPage);
using (var stream = response.Content.ReadAsStreamAsync().Result)
{
Console.WriteLine("Loading: {0}", sitemapPage);
XDocument sitemapPageContent = XDocument.Load(stream);
var urlsetElem = sitemapPageContent.Element(xmlns.GetName("urlset"));
var urlsElems = urlsetElem.Elements(xmlns.GetName("url"));
var appsLinks = urlsElems.Select(x => x.Element(xmlns.GetName("loc")))
.Select(x => x.Value);
foreach (var appLink in appsLinks)
{
var appId = appLink.Substring(appLink.LastIndexOf('/') + 1);
var culture = appLink.Substring(prefixLength,
appLink.IndexOf('/', prefixLength) -
prefixLength);
if (appId == "ROW")
{
continue;
}
globalAppsToCultures.AddOrUpdate(
appId,
new AppInfo2
{
Id = appId,
Url = appLink,
Countries = new List <string>(new[] { culture })
},
(c, ai) =>
{
ai.Countries.Add(culture);
return(ai);
}
);
}
Console.WriteLine("Total Found {0} applications", globalAppsToCultures.Count);
}
}
});
AppIds = new HashSet <string>(globalAppsToCultures.Keys);
if (LoadIdsOnly)
{
return;
}
var toProcess = globalAppsToCultures.AsEnumerable();
if (apps > 0)
{
toProcess = toProcess.Take(apps);
}
var toProcessList = toProcess.ToList();
var countdown = new CountdownEvent(toProcessList.Count);
var commands = new List <ICommandData>();
Parallel.ForEach(
toProcessList,
new ParallelOptions {
MaxDegreeOfParallelism = 8
},
(kvp, pls, index) =>
{
using (var session = Store.OpenSession())
{
try
{
Console.WriteLine("({0}/{1} - {2} remaining) Loading app {3}",
index, globalAppsToCultures.Count, countdown.CurrentCount,
kvp.Value.Id);
var existing = session.Load <App>("apps/" + kvp.Value.Id);
try
{
var app = GetAppDetails2(kvp.Value.Id, kvp.Value.Countries);
if (app == null)
{
return;
}
if (existing != null)
{
if ((app.Price != null && existing.Price != null &&
app.Price.Amount == existing.Price.Amount) ||
app.LastUpdated == existing.LastUpdated)
{
return;
}
app.FriendlyUrl = existing.FriendlyUrl;
app.PreviousPrice = existing.Price;
}
var putCommandData = new PutCommandData
{
Key = app.Id,
Document = RavenJObject.FromObject(app),
Metadata = new RavenJObject
{
{ "Raven-Entity-Name", new RavenJValue("Apps") },
{
"Raven-Clr-Type",
new RavenJValue("Win8Apps.Model.App, Win8Apps.Model")
}
}
};
lock (commands)
{
commands.Add(putCommandData);
}
}
catch (Exception ex)
{
}
}
finally
{
countdown.Signal();
}
}
});
countdown.Wait();
if (commands.Count > 0)
{
const int batchSize = 512;
int handled = 0;
do
{
var commandsToSave = commands
.Skip(handled)
.Take(batchSize)
.ToList();
Store.DatabaseCommands.Batch(commandsToSave);
handled += commandsToSave.Count();
Console.WriteLine("Saving {0} {1}/{2} applications", commandsToSave.Count(), handled, commands.Count);
Thread.Sleep(5000);
} while (handled < commands.Count);
}
}
// TODO: Move above into DetectedProc class methods
/// <summary>
/// Group address spaces into related buckets
///
/// We will assign an address space ID to each detected proc so we know what process belongs with who
/// After AS grouping we will know what EPTP belongs to which AS since one of the DP's will have it's CR3 in the VMCS
///
/// Yes it's a bit complicated.
///
/// The overall procedure however is straight forward in that;
///
/// * For every detected process
/// Bucket into groups which are the "Address spaces" that initially are
///
/// (a) based on kernel address space similarities
/// and then
/// (b) based on what VMCS value was found pointing to that group
///
/// This ensures that if we have several hypervisors with a possibly identical kernel grouping (i.e. the PFN's
/// were used by each kernel were identical), they are disambiguated by the VMCS. (Which can be validated later)
///
/// The benefit here is that brute forcing at this stage is fairly expensive and can lead to significant overhead, there does
/// tend to be some outliers for large systems that need to be looked at more to determine who they belong too. Nevertheless, it's
/// inconsequential if they are grouped with the appropriate AS since even if they are isolated into their own 'AS' this is an artificial
/// construct for our book keeping. The net result is that even if some process is grouped by itself due to some aggressive variation in
/// kernel PFN' use (lots of dual mapped memory/MDL's or something), it's still able to be dumped and analyzed.
///
///
/// </summary>
/// <param name="pTypes">Types to scan for, this is of the already detected processes list so it's already filtered really</param>
public void GroupAS(PTType pTypes = PTType.UNCONFIGURED)
{
var PT2Scan = pTypes == PTType.UNCONFIGURED ? PTType.ALL : pTypes;
//if (Phase >=3 && OverRidePhase)
// return;
// To join an AS group we want to see > 50% correlation which is a lot considering were only interoperating roughly 10-20 values (more like 12)
var p = from proc in Processes
where (((proc.PageTableType & PT2Scan) == proc.PageTableType))
orderby proc.CR3Value ascending
select proc;
ASGroups = new ConcurrentDictionary<int, ConcurrentBag<DetectedProc>>();
// we trim out the known recursive/self entries since they will naturally not be equivalent
var AlikelyKernelSet = from ptes in p.First().TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
int totUngrouped = Processes.Count();
int CurrASID = 1;
int LastGroupTotal = 0;
var grouped = new ConcurrentBag<DetectedProc>();
WriteLine($"Scanning for group correlations total processes {totUngrouped}");
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>();
while (true)
{
ForegroundColor = ConsoleColor.Yellow;
Parallel.ForEach(p, (proc) =>
{
var currKern = from ptes in proc.TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
var interSection = currKern.Intersect(AlikelyKernelSet);
var correlated = interSection.Count() * 1.00 / AlikelyKernelSet.Count();
// add this detected CR3/process address space to an address space grouping when
// the kernel range is above the acceptable threshold, the group does not contain this proc
// and this proc is not already joined into another group
if (correlated > GroupThreshold && !ASGroups[CurrASID].Contains(proc) && proc.AddressSpaceID == 0)
{
WriteColor(ConsoleColor.Cyan, $"MemberProces: Group {CurrASID} Type [{proc.PageTableType}] GroupCorrelation [{correlated:P3}] PID [{proc.CR3Value:X}]");
proc.AddressSpaceID = CurrASID;
ASGroups[CurrASID].Add(proc);
// global list to quickly scan
grouped.Add(proc);
}
});
ForegroundColor = ConsoleColor.Yellow;
var totGrouped = (from g in ASGroups.Values
select g).Sum(x => x.Count());
WriteLine($"Finished Group {CurrASID} collected size {ASGroups[CurrASID].Count()}, continuing to group");
// if there is more work todo, setup an entry for testing
if (totGrouped < totUngrouped)
{
// if we wind up here
// there has been no forward progress in isolating further groups
if(LastGroupTotal == totGrouped)
{
ForegroundColor = ConsoleColor.Red;
WriteLine($"Terminating with non-grouped process candidates. GroupThreshold may be too high. {GroupThreshold}");
var pz = from px in Processes
where px.AddressSpaceID == 0
select px;
// just add the ungrouped processes as a single each bare metal
// unless it has an existing VMCS pointer
foreach (var px in pz)
{
WriteLine(px);
CurrASID++;
px.AddressSpaceID = CurrASID;
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>() { px };
var isCandidate = from pvmcs in scan.HVLayer
where pvmcs.gCR3 == px.CR3Value
select pvmcs;
if (isCandidate.Count() > 0)
{
px.CandidateList = new List<VMCS>(isCandidate);
px.vmcs = px.CandidateList.First();
WriteColor( ConsoleColor.White, $"Detected ungrouped {px.CR3Value} as a candidate under {px.CandidateList.Count()} values (first){px.vmcs.EPTP}");
}
}
ForegroundColor = ConsoleColor.Yellow;
break;
}
CurrASID++;
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>();
WriteLine($"grouped count ({totGrouped}) is less than total process count ({totUngrouped}, rescanning...)");
LastGroupTotal = totGrouped;
}
else
break; // we grouped them all!
/// Isolate next un-grouped PageTable
var UnGroupedProc = from nextProc in Processes
where !grouped.Contains(nextProc)
select nextProc;
AlikelyKernelSet = from ptes in UnGroupedProc.First().TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
}
Console.WriteLine($"Done All process groups.");
// after grouping link VMCS back to the group who 'discovered' the VMCS in the first place!
var eptpz = VMCSs.Values.GroupBy(eptz => eptz.EPTP).OrderBy(eptx => eptx.Key).Select(ept => ept.First()).ToArray();
// find groups dominated by each vmcs
var VMCSGroup = from aspace in ASGroups.AsEnumerable()
from ept in eptpz
where aspace.Value.Any(adpSpace => adpSpace == ept.dp)
select new { AS = aspace, EPTctx = ept };
// link the proc back into the eptp
foreach (var ctx in VMCSGroup)
foreach (var dp in ctx.AS.Value)
{
if(dp.CandidateList == null)
dp.CandidateList = new List<VMCS>();
dp.vmcs = ctx.EPTctx;
dp.CandidateList.Add(ctx.EPTctx);
}
// resort by CR3
foreach (var ctx in ASGroups.Values)
{
var dpz = from d in ctx
orderby d.CR3Value descending
select d;
if (dpz.Count() >= 1)
{
var aspace = dpz.First().AddressSpaceID;
ASGroups[aspace] = new ConcurrentBag<DetectedProc>(dpz);
}
}
Phase = 4;
// were good, all Processes should have a VMCS if applicable and be identifiable by AS ID
}
public IEnumerable <KeyValuePair <string, ITimedTask> > GetAllTasks()
{
return(tasks.AsEnumerable());
}
/// <summary>
/// returns filtered and ordered list of games
/// </summary>
/// <param name="titleFilter"></param>
/// <param name="filter"></param>
/// <param name="order"></param>
/// <param name="ascending"></param>
/// <returns></returns>
public static string[] ListGames(string filterCommand, GameFilter filter, GameOrder order, bool ascending)
{
var games = Games.AsEnumerable();
if (!string.IsNullOrEmpty(filterCommand))
{
try
{
var commands = filterCommand.Trim().ToLower().Split(new char[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var command in commands)
{
if (command.StartsWith("*"))
{
// remove all white spaces in string
var c = Regex.Replace(command, @"\s+", string.Empty);
var op = GetOperatorFromCommand(c);
// DATE filters
if (c.Contains("*beaten"))
{
if (GetDateFromCommand(c.Replace($"*beaten{ op }", string.Empty), out int?year, out int?month, out int?day))
{
games = games.Where(g => EvaluateDates(g.Value.Completed, op, year, month, day));
Debug.WriteLine($"beaten{ op }{ year }/{ month }/{ day }");
}
}
else if (c.Contains("*added"))
{
if (GetDateFromCommand(c.Replace($"*added{ op }", string.Empty), out int?year, out int?month, out int?day))
{
games = games.Where(g => EvaluateDates(g.Value.Added, op, year, month, day));
Debug.WriteLine($"added{ op }{ year }/{ month }/{ day }");
}
}
else if (c.Contains("*played"))
{
if (GetDateFromCommand(c.Replace($"*played{ op }", string.Empty), out int?year, out int?month, out int?day))
{
games = games.Where(g => EvaluateDates(g.Value.DateTimesPlayed, op, year, month, day));
Debug.WriteLine($"played{ op }{ year }/{ month }/{ day }");
}
}
// NUMBER filters
else if (c.Contains("*playcount"))
{
if (GetNumberFromCommand(c.Replace($"*playcount{ op }", string.Empty), out int?count))
{
games = games.Where(g => EvaluateNum(g.Value.PlayCount, op, count.Value));
Debug.WriteLine($"playcount{ op }{ count }");
}
}
else if (c.Contains("*rating"))
{
if (GetNumberFromCommand(c.Replace($"*rating{ op }", string.Empty), out int?rating))
{
games = games.Where(g => EvaluateNum(g.Value.Rating, op, rating.Value));
Debug.WriteLine($"rating{ op }{ rating }");
}
}
}
else
{
// filter by game title
games = games.Where(g => g.Value.Title.ToLower().Contains(command));
}
}
}
catch (Exception ex)
{
Log.WriteLine(ex.ToString());
games = Games.AsEnumerable();
}
}
switch (filter)
{
case GameFilter.All: break;
case GameFilter.Installed: games = games.Where(g => !g.Value.Removed); break;
case GameFilter.Removed: games = games.Where(g => g.Value.Removed); break;
case GameFilter.Beaten: games = games.Where(g => g.Value.Completed.HasValue); break;
case GameFilter.Unbeaten: games = games.Where(g => !g.Value.Completed.HasValue && !g.Value.Removed); break;
case GameFilter.WithControllerSupport: games = games.Where(g => g.Value.GamepadFriendly == true && !g.Value.Removed); break;
case GameFilter.Unidentified: games = games.Where(g => g.Value.Image == null); break;
default:
throw new NotImplementedException(filter.ToString());
}
switch (order)
{
case GameOrder.Title: games = games.OrderBy(g => g.Value.Title); break;
case GameOrder.PlayTime: games = games.OrderBy(g => g.Value.TotalTimePlayed); break;
case GameOrder.PlayCount: games = games.OrderBy(g => g.Value.PlayCount); break;
case GameOrder.Rating: games = games.OrderBy(g => g.Value.Rating); break;
case GameOrder.BeatTime: games = games.OrderBy(g => g.Value.BeatenIn); break;
case GameOrder.DateAdded: games = games.OrderBy(g => g.Value.Added); break;
case GameOrder.LastPlayed:
{
games = games
.Where(g => !g.Value.LastPlayed.HasValue)
.OrderByDescending(g => g.Value.Added)
.Concat(games
.Where(g => g.Value.LastPlayed.HasValue)
.OrderBy(g => g.Value.LastPlayed));
}
break;
default:
throw new NotImplementedException(order.ToString());
}
var result = games.Select(g => g.Key);
if (!ascending)
{
result = result.Reverse();
}
return(result.ToArray());
}
private static IReadOnlyDictionary <WittyerType, IReadOnlyList <TResult> > ToReadOnly <TSource, TResult>(
ConcurrentDictionary <WittyerType, List <TSource> > variants) where TSource : TResult
=> new ReadOnlyDictionary <WittyerType, IReadOnlyList <TResult> >(variants.AsEnumerable()
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Cast <TResult>().ToReadOnlyList()));