private static async Task AddSourceTypesThatDeriveFromNameAsync(
Func <INamedTypeSymbol, SymbolSet, bool> typeMatches,
ConcurrentSet <SemanticModel> cachedModels,
SymbolSet typesToSearchFor,
ProjectIndex index,
SymbolSet result,
string name,
CancellationToken cancellationToken)
{
foreach (var(document, info) in index.NamedTypes[name])
{
cancellationToken.ThrowIfCancellationRequested();
var semanticModel = await document.GetRequiredSemanticModelAsync(cancellationToken).ConfigureAwait(false);
cachedModels.Add(semanticModel);
var resolvedType = info.TryResolve(semanticModel, cancellationToken);
if (resolvedType is INamedTypeSymbol namedType &&
typeMatches(namedType, typesToSearchFor))
{
result.Add(namedType);
}
}
}
public ExceptionlessElasticConfiguration(
AppOptions appOptions,
IQueue <WorkItemData> workItemQueue,
JsonSerializerSettings serializerSettings,
ICacheClient cacheClient,
IMessageBus messageBus,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory
) : base(workItemQueue, cacheClient, messageBus, loggerFactory)
{
_appOptions = appOptions;
_serializerSettings = serializerSettings;
_logger.LogInformation("All new indexes will be created with {ElasticsearchNumberOfShards} Shards and {ElasticsearchNumberOfReplicas} Replicas", _appOptions.ElasticsearchOptions.NumberOfShards, _appOptions.ElasticsearchOptions.NumberOfReplicas);
AddIndex(Stacks = new StackIndex(this));
AddIndex(Events = new EventIndex(this, serviceProvider, appOptions));
AddIndex(Migrations = new MigrationIndex(this, _appOptions.ElasticsearchOptions.ScopePrefix + "migrations", appOptions.ElasticsearchOptions.NumberOfReplicas));
AddIndex(Organizations = new OrganizationIndex(this));
AddIndex(Projects = new ProjectIndex(this));
AddIndex(Tokens = new TokenIndex(this));
AddIndex(Users = new UserIndex(this));
AddIndex(WebHooks = new WebHookIndex(this));
}
private static async Task AddDescendantSourceTypesInProjectAsync(
SymbolSet currentSourceAndMetadataTypes,
SymbolSet result,
Project project,
Func <INamedTypeSymbol, SymbolSet, bool> typeMatches,
Func <INamedTypeSymbol, bool> shouldContinueSearching,
bool transitive,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
// We're going to be sweeping over this project over and over until we reach a
// fixed point. In order to limit GC and excess work, we cache all the semantic
// models and DeclaredSymbolInfo for the documents we look at.
// Because we're only processing a project at a time, this is not an issue.
var cachedModels = new ConcurrentSet <SemanticModel>();
using var _1 = GetSymbolSet(out var typesToSearchFor);
using var _2 = GetSymbolSet(out var tempBuffer);
typesToSearchFor.AddAll(currentSourceAndMetadataTypes);
var projectIndex = await ProjectIndex.GetIndexAsync(project, cancellationToken).ConfigureAwait(false);
// As long as there are new types to search for, keep looping.
while (typesToSearchFor.Count > 0)
{
foreach (var type in typesToSearchFor)
{
cancellationToken.ThrowIfCancellationRequested();
switch (type.SpecialType)
{
case SpecialType.System_Object:
await AddMatchingTypesAsync(
cachedModels,
projectIndex.ClassesAndRecordsThatMayDeriveFromSystemObject,
result : tempBuffer,
predicateOpt : n => n.BaseType?.SpecialType == SpecialType.System_Object,
cancellationToken).ConfigureAwait(false);
break;
case SpecialType.System_ValueType:
await AddMatchingTypesAsync(
cachedModels,
projectIndex.ValueTypes,
result : tempBuffer,
predicateOpt : null,
cancellationToken).ConfigureAwait(false);
break;
case SpecialType.System_Enum:
await AddMatchingTypesAsync(
cachedModels,
projectIndex.Enums,
result : tempBuffer,
predicateOpt : null,
cancellationToken).ConfigureAwait(false);
break;
case SpecialType.System_MulticastDelegate:
await AddMatchingTypesAsync(
cachedModels,
projectIndex.Delegates,
result : tempBuffer,
predicateOpt : null,
cancellationToken).ConfigureAwait(false);
break;
}
await AddSourceTypesThatDeriveFromNameAsync(
typeMatches,
cachedModels,
typesToSearchFor,
projectIndex,
result : tempBuffer,
type.Name,
cancellationToken).ConfigureAwait(false);
}
PropagateTemporaryResults(
result, typesToSearchFor, tempBuffer, transitive, shouldContinueSearching);
}
}
static void Main(string[] args)
{
string documentpath = @"d:\requests\";
if (!Directory.Exists(documentpath))
{
Directory.CreateDirectory(documentpath);
}
string indicespath = @"d:\requests\indices";
if (!Directory.Exists(indicespath))
{
Directory.CreateDirectory(indicespath);
}
//CreateFiles();
DocumentIndexer indexer = new DocumentIndexer(documentpath);
Index<string> isMultiLead = null;
Index<string> clientName = null;
Index<int> projectId = null;
Index<string> isDotNet = null;
if (true)
{
Console.WriteLine("Rebuilding indices...");
isMultiLead = new DatafieldIndex("IsMultiLead", indicespath, documentpath, "isMultiLead");
clientName = new DatafieldIndex("ClientName", indicespath, documentpath, "clientname");
projectId = new ProjectIndex("ProjectId", indicespath, documentpath);
isDotNet = new DatafieldIndex("IsDotNet", indicespath, documentpath, "isDotNet");
indexer.Register(isMultiLead);
indexer.Register(clientName);
indexer.Register(isDotNet);
indexer.Register(projectId);
Stopwatch sw = new Stopwatch();
sw.Restart();
indexer.CreateForAll();
sw.Stop();
WriteInColor(string.Format("Project Index rebuilt in: {0}", sw.ElapsedMilliseconds), ConsoleColor.Yellow);
}
else
{
Console.WriteLine("Loading indices...");
Stopwatch sw = new Stopwatch();
sw.Restart();
isMultiLead = DatafieldIndex.Load(indicespath, "IsMultiLead").SetPropertyLocator((doc) =>
{
var elm = doc.Descendants("datafield").Where(i => i.Attribute("name").Value == "isMultiLead").FirstOrDefault();
string value = String.Empty;
if (elm != null)
{
value = elm.Value;
}
return value;
});
clientName = Index<string>.Load(indicespath, "ClientName").SetPropertyLocator((doc) =>
{
var elm = doc.Descendants("datafield").Where(i => i.Attribute("name").Value == "clientname").FirstOrDefault();
string value = String.Empty;
if (elm != null)
{
value = elm.Value;
}
return value;
});
projectId = Index<int>.Load(indicespath, "ProjectId")
.SetPropertyLocator((doc) =>
{
var value = int.Parse(doc.Root.Attribute("proj_id").Value);
return value;
});
isDotNet = Index<string>.Load(indicespath, "IsDotNet")
.SetPropertyLocator((doc) =>
{
var elm = doc.Descendants("datafield").Where(i => i.Attribute("name").Value == "isDotNet").FirstOrDefault();
string value = String.Empty;
if (elm != null)
{
value = elm.Value;
}
return value;
});
sw.Stop();
WriteInColor(string.Format("Indices loaded: {0}", sw.ElapsedMilliseconds), ConsoleColor.Yellow);
indexer.Register(isMultiLead);
indexer.Register(clientName);
indexer.Register(projectId);
indexer.Register(isDotNet);
}
//while (true)
//{
// Console.WriteLine("Press spacebar to search for multileads and project 465...");
// //Console.WriteLine("Press C to search on clientname...");
// Console.WriteLine("Press Backspace to quit...");
// ConsoleKeyInfo bla = Console.ReadKey(true);
// if (bla.Key == ConsoleKey.Spacebar)
// {
// DoTheSearch(isMultiLead, projectId);
// }
// //else if (bla.Key == ConsoleKey.C)
// //{
// // SearchClientName(clientName);
// //}
// else if (bla.Key == ConsoleKey.Backspace)
// {
// break;
// }
//}
Console.WriteLine("Search for non-DotNet of project id 220:");
Stopwatch sw2 = new Stopwatch();
sw2.Restart();
List<int> trues = isDotNet.Search(s => s == "false" || string.IsNullOrEmpty(s)).ToList();
List<int> projects = projectId.Search(s => s == 220).ToList();
var combined = trues.Intersect(projects);
sw2.Stop();
WriteInColor(string.Format("Requests found: {0}", combined.Count()), ConsoleColor.Red);
WriteInColor(string.Format("Ms elapsed: {0}", sw2.ElapsedMilliseconds), ConsoleColor.Yellow);
Console.WriteLine("Generating document");
Reporter reporter = new Reporter(documentpath, new List<string>() { "c_country", "n_country", "firstname", "clientname", "email","haschildren" , "tempstorage"}, @"d:\temp\requests.csv");
reporter.Create(combined);
Console.ReadKey();
}
