/// <summary>
/// Returns an object back into the pool. If the item is not returned to the pool and
/// it implements <see cref="IDisposable"/> then Dispose() will be called. If an exception
/// occurs in a <see cref="ReturnAction"/> the item is not returned to the pool and the item
/// is Disposed of if it implements <see cref="IDisposable"/> before the exception is rethrown.
/// </summary>
/// <param name="item">The item to release back into the pool.</param>
public void Return(T item)
{
// Don't allow multiple references to the same object to live on the Pool. We don't want to hand
// out what we think are unique object references and then find out they're being edited all over the
// place.
if (item == null || _items.Contains(item))
{
return;
}
// Only return the item the pool if the pool has spaces available.
if (_items.Count < this.Max)
{
try
{
// If this can't run, dispose of the item and then re-throw the exception
// so the caller can handle it as they see fit (and they know it occurred).
this.ReturnAction?.Invoke(item);
}
catch
{
DisposeItem(item);
throw;
}
_items.Add(item);
return;
}
// If item gets here it was not returned to the pool, as a result, if it needs
// to be disposed of then we're going to do that before it goes into the ether.
this.DisposeItem(item);
}
public void ConcurrentBagAddDistinctItems()
{
List <string> data = new List <string>();
var random = new Random();
for (int i = 0; i < 1000; i++)
{
data.Add(random.Next(1, 4).ToString());
}
Assert.AreEqual(1000, data.Count);
for (int i = 0; i < 10; i++)
{
ConcurrentBag <string> bag = new ConcurrentBag <string>();
data.AsParallel().ForAll(d =>
{
lock (data)
{
if (!bag.Contains(d))
{
Thread.Sleep(100);
bag.Add(d);
Thread.Sleep(100);
}
}
});
Assert.AreEqual(3, bag.Count);
}
}
private bool FileIsValid([NotNull] string file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
var includeExtensionList = _fileListFromPathFilter.FilterExtensionsToEqual ?? new List <string>();
var excludeExtensionList = _fileListFromPathFilter.FilterExtensionsNotToEqual ?? new List <string>();
var includeFileNameList = _fileListFromPathFilter.FilterFileNamesToEqual ?? new List <string>();
var excludeFileNameList = _fileListFromPathFilter.FilterFileNamesNotToEqual ?? new List <string>();
var fileInfo = new FileInfo(file);
var fileName = fileInfo.Name.ToLower();
var fileExtension = fileInfo.Extension.ToLower().TrimStart('.');
var alreadyContained = !_fileList.Contains(file);
var hasFileExtension = !string.IsNullOrWhiteSpace(fileExtension);
//!Any() => all allowed; else => list has to contain extension, name or path
var includeExtension = !includeExtensionList.Any() || includeExtensionList.Contains(fileExtension);
var includeFileName = !includeFileNameList.Any() || includeFileNameList.Contains(fileName);
// .docx
var excludeExtension = excludeExtensionList.Contains(fileExtension, StringComparer.InvariantCultureIgnoreCase);
// ...file.x
var excludeFileName = excludeFileNameList.Any(p => fileName.Contains(p, StringComparison.InvariantCultureIgnoreCase));
return(alreadyContained && hasFileExtension && includeExtension && !excludeExtension && includeFileName && !excludeFileName);
}
private bool ContainsOrganisationId(string orgId)
{
lock (_lockObject)
{
return(_organisationIdList != null && _organisationIdList.Contains(orgId));
}
}
private IList <string> ManifestHtmlsExceptTocHtmls(Manifest manifest, ConcurrentBag <string> tocHtmls)
{
var manifestConceptuals = GetManifestHtmls(manifest);
var others = manifestConceptuals.Where(p => !tocHtmls.Contains(p)).Distinct().ToList();
return(others);
}
public void AddObserver(ILogMessageObserver observer)
{
if (observer != null && !m_messageObservers.Contains(observer))
{
m_messageObservers.Add(observer);
}
}
public List <string> BuildProjects(List <DotNetProjectInfo> projects, string arguments)
{
var builtProjects = new ConcurrentBag <string>();
var totalProjectCountToBuild = projects.Count;
var buildingProjectIndex = 0;
try
{
foreach (var project in projects)
{
if (builtProjects.Contains(project.CsProjPath))
{
continue;
}
buildingProjectIndex++;
Console.WriteLine(
"Building....: " + " (" + buildingProjectIndex + "/" +
totalProjectCountToBuild + ")" + project.CsProjPath
);
BuildInternal(project, arguments, builtProjects);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
return(builtProjects.ToList());
}
public void PutObject(HealthRollup item)
{
if (!_concHealthRollColl.Contains(item))
{
_concHealthRollColl.Add(item);
}
}
/// <summary>
/// 保存数据库上下文
/// </summary>
/// <param name="dbContext"></param>
public void AddToPool(DbContext dbContext)
{
// 排除已经存在的数据库上下文
if (!dbContexts.Contains(dbContext))
{
dbContexts.Add(dbContext);
// 订阅数据库上下文操作失败事件
dbContext.SaveChangesFailed += (s, e) =>
{
if (!failedDbContexts.Contains(dbContext))
{
var context = s as DbContext;
// 当前事务
var currentTransaction = context.Database.CurrentTransaction;
if (currentTransaction != null)
{
// 回滚事务
currentTransaction.Rollback();
}
failedDbContexts.Add(context);
}
};
}
}
internal void AddSerializerObserver(IObserver <ISerializer> observer)
{
if (!_observers.Contains(observer))
{
_observers.Add(observer);
}
}
public virtual async Task DequeueMultipleItemsInParallelSucceeds()
{
// Arrange
var queue = (IBlockingQueue<T>)Create();
var items = new List<T>();;
var count = 100;
for (var i = 0; i < count; i++)
{
var item = Fixture.Create<T>();
items.Add(item);
await queue.EnqueueAsync(item);
}
var timeout = TimeSpan.FromMilliseconds(30000);
var cts = new CancellationTokenSource(timeout);
// Act
var actualItems = new ConcurrentBag<T>();
var tasks = Enumerable
.Range(0, count)
.Select(i => Task.Run(async () => actualItems.Add(await queue.DequeueAsync(cts.Token))));
await Task.WhenAll(tasks);
// Assert
AssertEquals(await queue.GetLengthAsync(), 0);
foreach (var item in items)
{
AssertIsTrue(actualItems.Contains(item));
}
}
void EnsureQueueInitialized(string queueName)
{
if (queuesAlreadyInitialized.Contains(queueName))
{
return;
}
var directory = GetDirectoryForQueueNamed(queueName);
if (Directory.Exists(directory))
{
return;
}
Exception caughtException = null;
try
{
Directory.CreateDirectory(directory);
}
catch (Exception exception)
{
caughtException = exception;
}
if (caughtException != null && !Directory.Exists(directory))
{
throw new ApplicationException(string.Format("Could not initialize directory '{0}' for queue named '{1}'", directory, queueName), caughtException);
}
// if an exception occurred but the directory exists now, it must have been a race... we're good
queuesAlreadyInitialized.Add(queueName);
}
public void verify_multithread_mapping()
{
for (int outerIteration = 0; outerIteration < 10; outerIteration++)
{
SetUp();
Int32 iteration = 0;
var sequence = Enumerable.Range(0, 100);
ConcurrentBag <String> generated = new ConcurrentBag <string>();
try
{
Parallel.ForEach(sequence, i =>
{
Interlocked.Increment(ref iteration);
generated.Add(sut.Map("TEST" + i));
});
Assert.That(generated.Count, Is.EqualTo(100), "Error in iteration " + outerIteration);
}
catch (Exception ex)
{
Assert.Fail("Exception at iteration " + iteration + ": " + ex.ToString());
}
var allRecords = mapperCollection.Find(Builders <BsonDocument> .Filter.Empty).ToList();
Assert.That(allRecords, Has.Count.EqualTo(100));
for (int i = 1; i <= 100; i++)
{
if (!generated.Contains("MapperTests_" + i))
{
Assert.Fail("Id " + i + " is missing");
}
}
Assert.That(generated.Distinct().Count(), Is.EqualTo(100));
}
}
internal static void Build(RuntimeTypeModel runtimeTypeModel, Type type)
{
if (BuiltTypes.Contains(type))
{
return;
}
lock (type)
{
if (runtimeTypeModel.CanSerialize(type))
{
if (type.IsGenericType)
{
BuildGenerics(runtimeTypeModel, type);
}
return;
}
var meta = runtimeTypeModel.Add(type, false);
var fields = type.GetFields(Flags);
meta.Add(fields.Select(x => x.Name).ToArray());
meta.UseConstructor = false;
BuildBaseClasses(runtimeTypeModel, type);
BuildGenerics(runtimeTypeModel, type);
foreach (var memberType in fields.Select(f => f.FieldType).Where(t => !t.IsPrimitive))
{
Build(runtimeTypeModel, memberType);
}
BuiltTypes.Add(type);
}
}
/// <summary>
/// Set all possible values to the OptionInput and use Levenshtein Diff to find best one
/// </summary>
/// <param name="inputModel"></param>
/// <param name="input"></param>
private void SetOptionInput(FormatOptionInputModel inputModel, string input, EditorStyles formatStyle, List <IFormatOption> formatOptions)
{
if (FormatOptionsInputValues.inputValues.ContainsKey(inputModel.Name) == false)
{
return;
}
if (customInput.Contains(inputModel.Name))
{
if (int.TryParse(inputModel.Input, out int result))
{
inputModel.Input = SetColumnTab(result, inputModel.Name);
}
}
else
{
var inputValuesLevenshtein = new Dictionary <string, int>();
string[] inputValues = FormatOptionsInputValues.inputValues[inputModel.Name];
var previousInput = inputModel.Input;
foreach (var item in inputValues)
{
inputModel.Input = item;
var levenshtein = GetLevenshteinAfterFormat(input, formatStyle, formatOptions);
inputValuesLevenshtein.Add(item, levenshtein);
}
var inputValue = inputValuesLevenshtein.OrderBy(e => e.Value).First();
inputModel.Input = inputValue.Value == inputValuesLevenshtein[previousInput] ?
previousInput : inputValue.Key;
}
}
/// <summary>
/// Walks the directory tree looking for the filename
/// </summary>
/// <param name="filename">the file to search for</param>
/// <param name="checkedPaths">the list of paths that have already been checked</param>
/// <param name="current">the current path that we are on</param>
/// <returns>the path to the filename if it is found</returns>
public static string DescendFileSystem(string filename, ConcurrentBag<string> checkedPaths, string current)
{
if (checkedPaths.Contains(current) || current.Contains("$RECYCLE.BIN"))
{
return string.Empty;
}
var file = Path.Combine(current, filename);
if (File.Exists(file))
{
return file;
}
checkedPaths.Add(current);
try
{
var paths = Directory.GetDirectories(current);
foreach (string path in paths)
{
var result = DescendFileSystem(filename, checkedPaths, path);
if (!string.IsNullOrEmpty(result))
{
return result;
}
}
}
catch (UnauthorizedAccessException)
{
return string.Empty;
}
return string.Empty;
}
public void Add(string token)
{
if (!concurrentBag.Contains(token))
{
concurrentBag.Add(token);
}
}
/// <summary>
/// Gets a <see cref="SportEventCI" /> instance representing cached sport event data
/// </summary>
/// <param name="id">A <see cref="URN" /> specifying the id of the sport event which cached representation to return</param>
/// <returns>a <see cref="SportEventCI" /> instance representing cached sport event data</returns>
public SportEventCI GetEventCacheItem(URN id)
{
Metric.Context("CACHE").Meter("SportEventCache->GetEventCacheItem", Unit.Calls);
lock (_addLock)
{
try
{
var item = (SportEventCI)Cache.Get(id.ToString());
if (item != null)
{
return(item);
}
item = _sportEventCacheItemFactory.Build(id);
AddNewCacheItem(item);
// if there are events for non-standard tournaments (tournaments not on All tournaments for all sports)
if (item is TournamentInfoCI && !SpecialTournaments.Contains(item.Id))
{
SpecialTournaments.Add(item.Id);
}
return(item);
}
catch (Exception ex)
{
ExecutionLog.Error($"Error getting cache item for id={id}", ex);
}
}
return(null);
}
/// <summary>
/// Try to generate an unique id.
/// </summary>
/// <param name="shape">The shape.</param>
/// <returns></returns>
public string GetUniqueId(OoShapeObserver shape)
{
int uidCount = 0;
String uid = String.Empty;
if (shape != null && shape.Shape != null)
{
if (OoUtils.ElementSupportsService(shape.Shape, OO.Services.DRAW_SHAPE_TEXT))
{
uid = shape.Name;
if (String.IsNullOrWhiteSpace(uid) ||
uid.Contains('\'') ||
uid.Contains(' '))
{
uid = shape.UINamePlural;
System.Diagnostics.Debug.WriteLine("______the start name for the text shape is now : '" + uid + "'");
}
}
else
{
uid = shape.Name;
}
if (String.IsNullOrWhiteSpace(uid))
{
if (shape is INameBuilder)
{
uid = ((INameBuilder)shape).BuildName();
}
else
{
uid = shape.UINameSingular + "_" + (++uidCount);
}
}
while (shapeIds.Contains(uid))
{
if (shape is INameBuilder)
{
uid = ((INameBuilder)shape).RebuildName();
}
else
{
int i_ = uid.LastIndexOf('_');
if (i_ >= 0)
{
uid = uid.Substring(0, i_ + 1);
}
else
{
uid += "_";
}
uid += ++uidCount;
}
}
shapeIds.Add(uid);
}
Logger.Instance.Log(LogPriority.DEBUG, this, "new Shape with name: " + uid + " registered");
return(uid);
}
public async Task OpenOrCloseMonitorDataSubscriptionAsync(KafkaOpenOrCloseMonitorDataInput input)
{
Console.WriteLine("这是FBoxMqttService的BoxOpenOrCloseDmonDataSubscriptionAsync方法:" +
JsonSerializer.Serialize(input));
var mqttManager = _mqttManagerFunc(input.MqttName);
// 获取当前开关点盒子topic
var topicNames = input.MonitorData.Select(y => y.BoxNo).Distinct().Select(boxNo =>
{
var topicPrefix = _mqttSetting.MqttTopicPrefix.Replace("${SN}", boxNo);
return(topicPrefix + _mqttSetting.PauseTopic);
});
IList <Task> mqttOperaTasks = new List <Task>();
if (input.Action == 0)
{
// 关点
foreach (var topicName in topicNames)
{
mqttOperaTasks.Add(mqttManager.PublishAsync(topicName, "Enable"));
}
await Task.WhenAll(mqttOperaTasks);
return;
}
// 开点,判断其当前盒子是否已被订阅监听数据,如果没有订阅,那么就进行订阅
// key为盒子号,value为订阅
var subMonitorTopicNames = new Dictionary <string, string>();
input.MonitorData.Select(y => y.BoxNo).Distinct().ToList().ForEach(boxNo =>
{
if (_boxOpenOrCloseMonitorDataBoxNoCache.Contains(boxNo))
{
return;
}
var topicPrefix = _mqttSetting.MqttTopicPrefix.Replace("${SN}", boxNo);
var topicName = topicPrefix + _mqttSetting.MonitorDataTopic;
subMonitorTopicNames.Add(boxNo, topicName);
});
if (subMonitorTopicNames.Any())
{
// 如果没有订阅,那么就进行订阅,并添加此盒子到缓存集合中
mqttOperaTasks.Add(mqttManager.SubscribeAsync(subMonitorTopicNames.Values.ToArray()));
foreach (var(key, _) in subMonitorTopicNames)
{
_boxOpenOrCloseMonitorDataBoxNoCache.Add(key);
}
mqttOperaTasks.Add(File.WriteAllTextAsync(
$"./Cache/{nameof(FBoxMqttService)}.{nameof(_boxOpenOrCloseMonitorDataBoxNoCache)}.txt",
JsonSerializer.Serialize(_boxOpenOrCloseMonitorDataBoxNoCache)));
}
foreach (var topicName in topicNames)
{
mqttOperaTasks.Add(mqttManager.PublishAsync(topicName, "Disable"));
}
await Task.WhenAll(mqttOperaTasks);
}
internal static void DoFilterAndAdd(List <JobItem> pageItems, ConcurrentBag <IDataItem> items, ServiceProvider sp)
{
WebClient wc = new WebClient();
foreach (var i in pageItems)
{
string lower = i.Name.ToLower();
bool exclude = false;
foreach (string ex in ExcludeKeywords)
{
if (lower.Contains(ex))
{
exclude = true;
break;
}
}
//Name has every thing
if (!exclude)
{
bool hasLocation = false;
foreach (string ex in IncludeKeywords)
{
if (lower.Contains(ex))
{
hasLocation = true;
break;
}
}
exclude = !hasLocation;
}
if (exclude)
{
continue;
}
lock (cache)
{
if (cache.Contains(i.ActionContent))
{
continue;
}
else
{
cache.Add(i.ActionContent);
}
}
if (NotContract(i, wc))
{
items.Add(i);
}
}
}
public EventSourcingGrain()
{
var assembly = this.GetType().Assembly;
if (!registerAssembly.Contains(assembly.FullName))
{
lock (lockObj)
{
if (!registerAssembly.Contains(assembly.FullName))
{
GrainInternalEventHandlerProvider.RegisterInternalEventHandler(assembly);
EventNameTypeMapping.RegisterEventType(assembly);
registerAssembly.Add(assembly.FullName);
}
}
}
}
/// <summary>
/// 保存数据库上下文
/// </summary>
/// <param name="dbContext"></param>
public void AddToPool(DbContext dbContext)
{
// 排除已经存在的数据库上下文
if (!dbContexts.Contains(dbContext))
{
dbContexts.Add(dbContext);
}
}
public static bool IsValidKey(string key)
{
if (_apiKeys.Contains(key))
{
return(true);
}
return(false);
}
private void AddReg(string name)
{
if (regs.Contains(name))
{
return;
}
regs.Add(name);
}
public void Subscribe(ISubscriberMoney subscriber)
{
if (!_subscribers.Contains(subscriber))
{
Output.EscreverTexto($"Mais um caboclo inscrito, nome: {subscriber.ToString()}");
_subscribers.Add(subscriber);
}
}
public static void RegisterServiceHost(ServiceConfiguration serviceHost)
{
if (CachedIssuers.Contains(serviceHost))
{
return;
}
CachedIssuers.Add(serviceHost);
}
public async Task RemoveAccountFromBankSever(ConcurrentBag <Account> l, Account ac)
{
if (l.Contains(ac))
{
// l.a;
}
await Task.Delay(2000);
}
protected override void OnServerConnected(string serverName)
{
if (!_serversToInitialize.Contains(serverName))
{
_serversToInitialize.Add(serverName);
}
_statusUpdateHandle.Set();
}
public void EnqueueIfNotProcessed(string url)
{
if (!_processedQueues.Contains(url))
{
_urlsQueue.Enqueue(url);
_processedQueues.Add(url);
}
}
public override void ProcessFile(ConcurrentBag <int> missing, string fileToProcess, string fieldToUpdate, EdgarTaskState state, string[] allLines, string header, string cacheFolder, string tsvFileName, bool processInParallel)
{
//https://msdn.microsoft.com/en-us/library/ex21zs8x(v=vs.110).aspx
//https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/transaction-and-bulk-copy-operations
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " -- BEGIN BULK PROCESS");
using (IEdgarDatasetsBulkRepository repo = new EdgarDatasetsBulkRepository())
{
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " -- Initializing variables");
DataTable dt = repo.GetEmptyDataTable(RelatedTable);
List <string> fieldNames = header.Split('\t').ToList();
ConcurrentDictionary <int, string> failedLines = new ConcurrentDictionary <int, string>();
List <Exception> exceptions = new List <Exception>();
int lineNumber = 0;
string prefixMsg = "Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess;
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " -- Creating DataTable");
//first line is the header
for (int i = 1; i < allLines.Length; i++)
{
lineNumber = i + 1;//i+1: indexes starts with 0 but header is line 1 and the first row is line 2
//It will be processed if:
//it's the first time (missing == null)
//or it's processed again and line wasn't processed the firs time (missing.Contains(i+1))
if (missing == null || missing.Contains(i + 1))
{
string line = allLines[i];
if (!string.IsNullOrEmpty(line))
{
try
{
List <string> fields = line.Split('\t').ToList();
DataRow dr = dt.NewRow();
Parse(fieldNames, fields, i + 1, dr, state.Dataset.Id);
dt.Rows.Add(dr);
}
catch (Exception ex)
{
EdgarLineException elex = new EdgarLineException(fileToProcess, lineNumber, ex);
exceptions.Add(elex);
failedLines.TryAdd(lineNumber, line);
Log.Error(prefixMsg + " -- line[" + lineNumber.ToString() + "]: " + line);
Log.Error(prefixMsg + " -- line[" + lineNumber.ToString() + "]: " + ex.Message, elex);
if (exceptions.Count > MaxErrorsAllowed)
{
Log.Fatal(prefixMsg + " -- line[" + i.ToString() + "]: max errors allowed reached", ex);
throw new EdgarDatasetException(fileToProcess, exceptions);
}
}
}
}
}
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " -- Starting bulk copy");
repo.BulkCopyTable(RelatedTable, dt);
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " --End bulk copy, now saving failed lines.");
state.FileNameToReprocess = WriteFailedLines(cacheFolder, tsvFileName, header, failedLines, allLines.Length);
Log.Info("Datasetid " + state.Dataset.Id.ToString() + " -- " + fileToProcess + " -- END BULK PROCESS");
}
}
public void MultipleInsertTest()
{
var modelReferences = new ConcurrentBag<IModelReference>();
for (int i = 0; i < 1000; i++)
{
var modelReference = this.TrackDao.InsertTrack(new TrackData("isrc", "artist", "title", "album", 2012, 200)
{
GroupId = "group-id"
});
Assert.IsFalse(modelReferences.Contains(modelReference));
modelReferences.Add(modelReference);
}
}
public void ShouldInsertMultipleTracksConcurrently()
{
const int NumberOfTracks = 1000;
var modelReferences = new ConcurrentBag<IModelReference>();
for (int i = 0; i < NumberOfTracks; i++)
{
var modelReference =
trackDao.InsertTrack(
new TrackData("isrc", "artist", "title", "album", 2012, 200) { GroupId = "group-id" });
Assert.IsFalse(modelReferences.Contains(modelReference));
modelReferences.Add(modelReference);
}
Assert.AreEqual(NumberOfTracks, trackDao.ReadAll().Count);
}
/// <summary>
/// ReadPackagesFromDisk loads all packages from disk and determines additional metadata such as the hash, IsAbsoluteLatestVersion, and IsLatestVersion.
/// </summary>
private HashSet<ServerPackage> ReadPackagesFromDisk()
{
_logger.Log(LogLevel.Info, "Start reading packages from disk...");
MonitorFileSystem(false);
try
{
var cachedPackages = new ConcurrentBag<ServerPackage>();
bool enableDelisting = EnableDelisting;
var packages = _expandedPackageRepository.GetPackages().ToList();
Parallel.ForEach(packages, package =>
{
// Create server package
var serverPackage = CreateServerPackage(package, enableDelisting);
// Add the package to the cache, it should not exist already
if (cachedPackages.Contains(serverPackage))
{
_logger.Log(LogLevel.Warning, "Duplicate package found - {0} {1}", package.Id, package.Version);
}
else
{
cachedPackages.Add(serverPackage);
}
});
_logger.Log(LogLevel.Info, "Finished reading packages from disk.");
return new HashSet<ServerPackage>(cachedPackages, PackageEqualityComparer.IdAndVersion);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, "Error while reading packages from disk: {0} {1}", ex.Message, ex.StackTrace);
throw;
}
finally
{
MonitorFileSystem(true);
}
}
/// <summary>
/// Adds the type to work list.
/// </summary>
/// <param name="messageType">Type of the message.</param>
private void AddTypeToWorkList(Type messageType)
{
ConcurrentBag<Type> listout;
if (!_workList.TryGetValue(messageType, out listout))
{
foreach (var pair in _handlerList)
{
if (pair.Key.IsAssignableFrom(messageType))
{
var list = new ConcurrentBag<Type>();
list = _workList.GetOrAdd(messageType, list);
Parallel.ForEach(pair.Value, t =>
{
_monitor.WaitOne();
if (!list.Contains(t))
{
list.Add(t);
}
_monitor.Set();
});
}
}
}
}
private static void CheckQueueMessage(string message1, ConcurrentBag<string> recievedLogsGetParam1)
{
var success = recievedLogsGetParam1.Contains(message1);
Assert.True(success, string.Format("message '{0}' not found", message1));
}
public async Task<List<TimeSpan>> Run()
{
var results = new List<TimeSpan>();
for (var n = 0; n < NoOfIterations; n++)
{
DateTime start = DateTime.Now;
var proposers = new ConcurrentBag<Proposer>();
var acceptors = new ConcurrentBag<Acceptor>();
var proposedValues = new ConcurrentBag<string>();
var acceptedValues = new ConcurrentBag<Task<string>>();
for (int i = 0; i < Proposers; i++)
{
proposers.Add(new Proposer(Guid.NewGuid().ToString(), Acceptors));
}
for (int i = 0; i < LiveAcceptors; i++)
{
acceptors.Add(new Acceptor("Address"));
}
foreach (var proposer in proposers)
foreach (var acceptor in acceptors)
proposer.Pipe(new Delay<NetworkMessage>(TimeSpan.FromMilliseconds(minDelay), TimeSpan.FromMilliseconds(maxDelay)))
.Pipe(new Drop<NetworkMessage>(drop))
.Pipe(acceptor)
.Pipe(new Delay<NetworkMessage>(TimeSpan.FromMilliseconds(minDelay), TimeSpan.FromMilliseconds(maxDelay)))
.Pipe(new Drop<NetworkMessage>(drop))
.Pipe(proposer);
foreach (var proposer in proposers)
{
acceptedValues.Add(Task.Factory.StartNew(() =>
{
var val = Guid.NewGuid().ToString();
proposedValues.Add(val);
return proposer.Propose(val);
})
.Unwrap());
}
var acceptedValue = await acceptedValues.First();
foreach (var res in acceptedValues)
{
var result = await res;
if (result != acceptedValue) throw new Exception("The proposers did not all get the same result");
if (!proposedValues.Contains(result)) throw new Exception("The accepted Value was never proposed");
}
DateTime end = DateTime.Now;
results.Add(end.Subtract(start));
}
return results;
}
public async Task LoadUsersSequentialAsync(List<string> userIds)
{
bool retval = true;
ConcurrentBag<string> usersFailedToLoad = new ConcurrentBag<string>();
List<Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser>>> userLoadTasks = new
List<Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser>>>();
Dictionary<int, string> loadTaskIds = new Dictionary<int, string>();
DateTime start = DateTime.Now;
foreach (string userId in userIds)
{
Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser>> loadTask = connector.GetFrameworkUserAsync(userId, true);
userLoadTasks.Add(loadTask);
loadTaskIds.Add(loadTask.Id, userId);
}
try
{
await Task.WhenAll(userLoadTasks).ConfigureAwait(false);
}
catch (Exception e)
{
int nbCompleted = userLoadTasks.Where(t => t.IsCompleted).Count();
int nbFaulted = userLoadTasks.Where(t => t.IsFaulted).Count();
log("Exception when loading framework users from server " + ServerURL + ": " + e.Message + ", nb completed " + nbCompleted + " nb faulted "
+ nbFaulted, 2);
StringBuilder sb = new StringBuilder();
if (nbFaulted > 0)
sb.Append("Dumping errors of faulted tasks: ");
foreach (Task t in userLoadTasks.Where(t => t.IsFaulted))
{
sb.Append("User load task for user " + loadTaskIds[t.Id] + " failed: ");
foreach (Exception x in t.Exception.Flatten().InnerExceptions)
{
sb.Append(x.Message + " at " + x.StackTrace);
sb.Append(Environment.NewLine);
}
}
if (sb.Length > 0)
log(sb.ToString(), 2);
}
log("Waiting for user load tasks complete, duration " + DateTime.Now.Subtract(start).TotalMilliseconds + " ms", 4);
List<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser> usersToLoadOts = new List<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser>();
foreach (Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser>> loadTask in userLoadTasks.Where(t => t.IsCompleted))
{
AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser> userRes = loadTask.Result;
if (!userRes.Success)
{
string userId = loadTaskIds[loadTask.Id];
log("Unable to extract user " + userId + " from server, " + ServerURL, 2);
usersFailedToLoad.Add(userId);
retval = false;
}
else
{
if (connector.HasOtsLine(userRes.ResultObject))
{
usersToLoadOts.Add(userRes.ResultObject);
}
}
}
if (userLoadTasks.Where(t => !t.IsCompleted).Count() > 0)
retval = false;
List<Task> dataLoadTasks = new List<Task>();
List<Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcOtsAccount>>> otsLoadTasks =
new List<Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcOtsAccount>>>();
List<Task<AlcPhoneOperationResult<VoiceMailInfo>>> vmLoadTasks = new List<Task<AlcPhoneOperationResult<VoiceMailInfo>>>();
Dictionary<int, string> otsLoadTaskIds = new Dictionary<int, string>();
Dictionary<int, string> vmLoadTaskIds = new Dictionary<int, string>();
foreach (AlcatelXmlApi6.AlcFwManagement.AlcFrameworkUser user in usersToLoadOts)
{
Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcOtsAccount>> otsTask =
connector.GetOtsAccountAsync(user.companyContacts.officePhone, true);
dataLoadTasks.Add(otsTask);
otsLoadTaskIds.Add(otsTask.Id, user.loginName);
otsLoadTasks.Add(otsTask);
Task<AlcPhoneOperationResult<VoiceMailInfo>> vmTask = connector.GetVoiceMailInfoAsync(user.companyContacts.officePhone);
dataLoadTasks.Add(vmTask);
vmLoadTaskIds.Add(vmTask.Id, user.loginName);
vmLoadTasks.Add(vmTask);
}
try
{
await Task.WhenAll(dataLoadTasks).ConfigureAwait(false);
}
catch (Exception e)
{
int nbCompleted = dataLoadTasks.Where(t => t.IsCompleted).Count();
int nbFaulted = dataLoadTasks.Where(t => t.IsFaulted).Count();
log("Exception when loading ots data / vm data from server " + ServerURL + ": " + e.Message + ", nb completed " + nbCompleted + " nb faulted "
+ nbFaulted, 2);
StringBuilder sb = new StringBuilder();
if (nbFaulted > 0)
sb.Append("Dumping errors of faulted tasks: ");
foreach (Task t in dataLoadTasks.Where(t => t.IsFaulted))
{
if (otsLoadTaskIds.ContainsKey(t.Id))
{
sb.Append("ots load task for user " + otsLoadTaskIds[t.Id] + " failed: ");
if (!usersFailedToLoad.Contains(otsLoadTaskIds[t.Id]))
usersFailedToLoad.Add(otsLoadTaskIds[t.Id]);
}
else
{
sb.Append("vminfo load task for user " + vmLoadTaskIds[t.Id] + " failed: ");
if (!usersFailedToLoad.Contains(vmLoadTaskIds[t.Id]))
usersFailedToLoad.Add(vmLoadTaskIds[t.Id]);
}
foreach (Exception x in t.Exception.Flatten().InnerExceptions)
{
sb.Append(x.Message + " at " + x.StackTrace);
sb.Append(Environment.NewLine);
}
}
if (sb.Length > 0)
log(sb.ToString(), 2);
}
TimeSpan duration = DateTime.Now.Subtract(start);
foreach (Task<AlcFrameworkUserOperationResult<AlcatelXmlApi6.AlcFwManagement.AlcOtsAccount>> otsTask in
otsLoadTasks.Where(t => t.Status == TaskStatus.RanToCompletion))
{
if (!otsTask.Result.Success)
{
log("Unable to load ots data of user " + otsLoadTaskIds[otsTask.Id] + ": " + otsTask.Result.ToString(), 2);
if (!usersFailedToLoad.Contains(otsLoadTaskIds[otsTask.Id]))
usersFailedToLoad.Add(otsLoadTaskIds[otsTask.Id]);
}
}
foreach (Task<AlcPhoneOperationResult<VoiceMailInfo>> vmLoadTask in vmLoadTasks.Where(t => t.Status == TaskStatus.RanToCompletion))
{
if (!vmLoadTask.Result.Success)
{
log("Unable to load voicemail data of user " + vmLoadTasks[vmLoadTask.Id] + ": " + vmLoadTask.Result.ToString(), 2);
if (!usersFailedToLoad.Contains(vmLoadTaskIds[vmLoadTask.Id]))
usersFailedToLoad.Add(vmLoadTaskIds[vmLoadTask.Id]);
}
}
if (dataLoadTasks.Where(t => !t.IsCompleted).Count() > 0)
retval = false;
log("Duration for sequential async user loading: " + duration.TotalMilliseconds, 4);
log("Result of framework user loading: " + retval + " nb failed " + usersFailedToLoad.Count, 4);
}
public void ShouldCatchAndHandleExceptionsThrownByEndpointObservables()
{
FakeServiceEndpoint erroringEndpoint = new FakeServiceEndpoint(typeof(ITestServiceMessage1)) { ThrowException = true };
FakeServiceEndpoint serviceEndpoint = new FakeServiceEndpoint(typeof(ITestServiceMessage2));
IServiceBus serviceBus = ServiceBus.Configure()
.WithEndpoint((IServiceEndpointClient) erroringEndpoint)
.WithEndpoint((IServiceEndpoint)erroringEndpoint)
.WithEndpoint((IServiceEndpointClient) serviceEndpoint)
.WithEndpoint((IServiceEndpoint)serviceEndpoint)
.UsingConsoleLogging().Create();
ConcurrentBag<IMessage> messages = new ConcurrentBag<IMessage>();
ConcurrentBag<Exception> exceptions = new ConcurrentBag<Exception>();
serviceBus.Events.Subscribe(messages.Add);
serviceBus.Commands.Subscribe(messages.Add);
serviceBus.Requests.Subscribe(messages.Add);
serviceBus.Exceptions.Subscribe(exceptions.Add);
// trigger exception
serviceBus.PublishAsync(new TestServiceEvent1());
TestServiceEvent2 message1 = new TestServiceEvent2();
serviceBus.PublishAsync(message1);
// trigger another exception
serviceBus.PublishAsync(new TestServiceEvent1());
TestServiceEvent2 message2 = new TestServiceEvent2();
serviceBus.PublishAsync(message2);
Assert.That(exceptions.Count(), Is.EqualTo(2));
Assert.That(messages.Contains(message1), "message1 not received");
Assert.That(messages.Contains(message2), "message2 not received");
}
public void Contains()
{
var TestObject = new ConcurrentBag<int>(new int[] { 1, 2, 3, 4, 5, 6 });
Assert.True(TestObject.Contains(4));
Assert.False(TestObject.Contains(-1));
}
// TODO: Move above into DetectedProc class methods
/// <summary>
/// Group address spaces into related buckets
///
/// We will assign an address space ID to each detected proc so we know what process belongs with who
/// After AS grouping we will know what EPTP belongs to which AS since one of the DP's will have it's CR3 in the VMCS
///
/// Yes it's a bit complicated.
///
/// The overall procedure however is straight forward in that;
///
/// * For every detected process
/// Bucket into groups which are the "Address spaces" that initially are
///
/// (a) based on kernel address space similarities
/// and then
/// (b) based on what VMCS value was found pointing to that group
///
/// This ensures that if we have several hypervisors with a possibly identical kernel grouping (i.e. the PFN's
/// were used by each kernel were identical), they are disambiguated by the VMCS. (Which can be validated later)
///
/// The benefit here is that brute forcing at this stage is fairly expensive and can lead to significant overhead, there does
/// tend to be some outliers for large systems that need to be looked at more to determine who they belong too. Nevertheless, it's
/// inconsequential if they are grouped with the appropriate AS since even if they are isolated into their own 'AS' this is an artificial
/// construct for our book keeping. The net result is that even if some process is grouped by itself due to some aggressive variation in
/// kernel PFN' use (lots of dual mapped memory/MDL's or something), it's still able to be dumped and analyzed.
///
///
/// </summary>
/// <param name="pTypes">Types to scan for, this is of the already detected processes list so it's already filtered really</param>
public void GroupAS(PTType pTypes = PTType.UNCONFIGURED)
{
var PT2Scan = pTypes == PTType.UNCONFIGURED ? PTType.ALL : pTypes;
//if (Phase >=3 && OverRidePhase)
// return;
// To join an AS group we want to see > 50% correlation which is a lot considering were only interoperating roughly 10-20 values (more like 12)
var p = from proc in Processes
where (((proc.PageTableType & PT2Scan) == proc.PageTableType))
orderby proc.CR3Value ascending
select proc;
ASGroups = new ConcurrentDictionary<int, ConcurrentBag<DetectedProc>>();
// we trim out the known recursive/self entries since they will naturally not be equivalent
var AlikelyKernelSet = from ptes in p.First().TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
int totUngrouped = Processes.Count();
int CurrASID = 1;
int LastGroupTotal = 0;
var grouped = new ConcurrentBag<DetectedProc>();
WriteLine($"Scanning for group correlations total processes {totUngrouped}");
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>();
while (true)
{
ForegroundColor = ConsoleColor.Yellow;
Parallel.ForEach(p, (proc) =>
{
var currKern = from ptes in proc.TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
var interSection = currKern.Intersect(AlikelyKernelSet);
var correlated = interSection.Count() * 1.00 / AlikelyKernelSet.Count();
// add this detected CR3/process address space to an address space grouping when
// the kernel range is above the acceptable threshold, the group does not contain this proc
// and this proc is not already joined into another group
if (correlated > GroupThreshold && !ASGroups[CurrASID].Contains(proc) && proc.AddressSpaceID == 0)
{
WriteColor(ConsoleColor.Cyan, $"MemberProces: Group {CurrASID} Type [{proc.PageTableType}] GroupCorrelation [{correlated:P3}] PID [{proc.CR3Value:X}]");
proc.AddressSpaceID = CurrASID;
ASGroups[CurrASID].Add(proc);
// global list to quickly scan
grouped.Add(proc);
}
});
ForegroundColor = ConsoleColor.Yellow;
var totGrouped = (from g in ASGroups.Values
select g).Sum(x => x.Count());
WriteLine($"Finished Group {CurrASID} collected size {ASGroups[CurrASID].Count()}, continuing to group");
// if there is more work todo, setup an entry for testing
if (totGrouped < totUngrouped)
{
// if we wind up here
// there has been no forward progress in isolating further groups
if(LastGroupTotal == totGrouped)
{
ForegroundColor = ConsoleColor.Red;
WriteLine($"Terminating with non-grouped process candidates. GroupThreshold may be too high. {GroupThreshold}");
var pz = from px in Processes
where px.AddressSpaceID == 0
select px;
// just add the ungrouped processes as a single each bare metal
// unless it has an existing VMCS pointer
foreach (var px in pz)
{
WriteLine(px);
CurrASID++;
px.AddressSpaceID = CurrASID;
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>() { px };
var isCandidate = from pvmcs in scan.HVLayer
where pvmcs.gCR3 == px.CR3Value
select pvmcs;
if (isCandidate.Count() > 0)
{
px.CandidateList = new List<VMCS>(isCandidate);
px.vmcs = px.CandidateList.First();
WriteColor( ConsoleColor.White, $"Detected ungrouped {px.CR3Value} as a candidate under {px.CandidateList.Count()} values (first){px.vmcs.EPTP}");
}
}
ForegroundColor = ConsoleColor.Yellow;
break;
}
CurrASID++;
ASGroups[CurrASID] = new ConcurrentBag<DetectedProc>();
WriteLine($"grouped count ({totGrouped}) is less than total process count ({totUngrouped}, rescanning...)");
LastGroupTotal = totGrouped;
}
else
break; // we grouped them all!
/// Isolate next un-grouped PageTable
var UnGroupedProc = from nextProc in Processes
where !grouped.Contains(nextProc)
select nextProc;
AlikelyKernelSet = from ptes in UnGroupedProc.First().TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
}
Console.WriteLine($"Done All process groups.");
// after grouping link VMCS back to the group who 'discovered' the VMCS in the first place!
var eptpz = VMCSs.Values.GroupBy(eptz => eptz.EPTP).OrderBy(eptx => eptx.Key).Select(ept => ept.First()).ToArray();
// find groups dominated by each vmcs
var VMCSGroup = from aspace in ASGroups.AsEnumerable()
from ept in eptpz
where aspace.Value.Any(adpSpace => adpSpace == ept.dp)
select new { AS = aspace, EPTctx = ept };
// link the proc back into the eptp
foreach (var ctx in VMCSGroup)
foreach (var dp in ctx.AS.Value)
{
if(dp.CandidateList == null)
dp.CandidateList = new List<VMCS>();
dp.vmcs = ctx.EPTctx;
dp.CandidateList.Add(ctx.EPTctx);
}
// resort by CR3
foreach (var ctx in ASGroups.Values)
{
var dpz = from d in ctx
orderby d.CR3Value descending
select d;
if (dpz.Count() >= 1)
{
var aspace = dpz.First().AddressSpaceID;
ASGroups[aspace] = new ConcurrentBag<DetectedProc>(dpz);
}
}
Phase = 4;
// were good, all Processes should have a VMCS if applicable and be identifiable by AS ID
}
