private DslContainer Initialize(DslContainer dslContainer)
{
var swTotal = Stopwatch.StartNew();
var nodes = _dslParser.GetConcepts();
var swConvert = Stopwatch.StartNew();
var parsedConcepts = ConceptInfoHelper.ConvertNodesToConceptInfos(nodes);
_performanceLogger.Write(swConvert, nameof(ConceptInfoHelper.ConvertNodesToConceptInfos));
var alternativeInitializationGeneratedReferences = InitializeAlternativeInitializationConcepts(parsedConcepts);
var swFirstAdd = Stopwatch.StartNew();
dslContainer.AddNewConceptsAndReplaceReferences(new[] { CreateInitializationConcept() });
dslContainer.AddNewConceptsAndReplaceReferences(parsedConcepts);
dslContainer.AddNewConceptsAndReplaceReferences(alternativeInitializationGeneratedReferences);
_performanceLogger.Write(swFirstAdd, $"Initialize: First AddNewConceptsAndReplaceReferences ({dslContainer.Concepts.Count()} concepts).");
ExpandMacroConcepts(dslContainer);
CheckSemantics(dslContainer);
dslContainer.SortReferencesBeforeUsingConcept(_buildOptions.InitialConceptsSort);
ReportObsoleteConcepts(dslContainer);
_dslModelFile.SaveConcepts(dslContainer.Concepts);
_performanceLogger.Write(swTotal, $"Initialize ({dslContainer.Concepts.Count()} concepts).");
return(dslContainer);
}
public DslModel(IDslParser dslParser, ILogProvider logProvider)
{
_dslParser = dslParser;
_performanceLogger = logProvider.GetLogger("Performance");
_logger = logProvider.GetLogger("DslModel");
_dslContainer = new DslContainer(logProvider);
}
private void ReportObsoleteConcepts(DslContainer dslContainer)
{
var obsoleteConceptsByType = dslContainer.Concepts
.GroupBy(concept => concept.GetType())
.Select(conceptsGroup => new
{
ConceptType = conceptsGroup.Key,
ConceptKeyword = ConceptInfoHelper.GetKeywordOrTypeName(conceptsGroup.Key),
Concepts = conceptsGroup.ToList(),
ObsoleteAttribute = (ObsoleteAttribute)conceptsGroup.Key.GetCustomAttributes(typeof(ObsoleteAttribute), false).SingleOrDefault()
})
.Where(conceptsGroup => conceptsGroup.ObsoleteAttribute != null)
.ToList();
// Obsolete concepts in the report are grouped by concept keyword and obsolete message.
var obsoleteConceptsByUserReport = obsoleteConceptsByType
.GroupBy(conceptsGroup => new { conceptsGroup.ConceptKeyword, conceptsGroup.ObsoleteAttribute.Message })
.Select(conceptsGroup => new
{
conceptsGroup.Key.ConceptKeyword,
ObsoleteMessage = conceptsGroup.Key.Message,
Concepts = conceptsGroup.SelectMany(group => group.Concepts)
})
.ToList();
foreach (var conceptsGroup in obsoleteConceptsByUserReport)
{
_logger.Warning(() => string.Format("Obsolete concept {0} ({1} occurrences). {2}",
conceptsGroup.Concepts.First().GetUserDescription(),
conceptsGroup.Concepts.Count(),
conceptsGroup.ObsoleteMessage));
}
}
private void CheckSemantics(DslContainer dslContainer)
{
var sw = Stopwatch.StartNew();
// Validations are grouped by concept type, for group performance diagnostics.
var validationsByConcept = new MultiDictionary <Type, Action>();
foreach (var conceptValidation in dslContainer.FindByType <IValidatedConcept>())
{
validationsByConcept.Add(conceptValidation.GetType(), () => conceptValidation.CheckSemantics(dslContainer));
}
var validationStopwatches = new Dictionary <Type, Stopwatch>();
foreach (var validationsGroup in validationsByConcept)
{
var validationStopwatch = Stopwatch.StartNew();
foreach (var validation in validationsGroup.Value)
{
validation.Invoke();
}
validationStopwatch.Stop();
validationStopwatches.Add(validationsGroup.Key, validationStopwatch);
}
foreach (var validationStopwatch in validationStopwatches.OrderByDescending(vsw => vsw.Value.Elapsed.TotalSeconds).Take(3))
{
_performanceLogger.Write(validationStopwatch.Value, () => "CheckSemantics total time for " + validationStopwatch.Key.Name + ".");
}
_performanceLogger.Write(sw, "CheckSemantics");
}
public DslModelFile(
ILogProvider logProvider,
DslContainer dslContainer,
ISqlExecuter sqlExecuter)
{
_performanceLogger = logProvider.GetLogger("Performance");
_dslContainer = dslContainer;
}
private string LogCreatedConcepts(DslContainer dslContainer, IEnumerable <IConceptInfo> macroCreatedConcepts, List <IConceptInfo> newUniqueConcepts)
{
var report = new StringBuilder();
LogConcepts(report, "Macro created", macroCreatedConcepts, first: true);
LogConcepts(report, "New unique", newUniqueConcepts);
LogConcepts(report, "New unresolved", newUniqueConcepts.Where(c => dslContainer.FindByKey(c.GetKey()) == null));
return(report.ToString());
}
public DslModelFile(
ILogProvider logProvider,
DslContainer dslContainer,
IAssetsOptions assetsOptions)
{
_performanceLogger = logProvider.GetLogger("Performance." + GetType().Name);
_dslContainer = dslContainer;
_rhetosEnvironment = assetsOptions;
}
public DslModelFile(
ILogProvider logProvider,
DslContainer dslContainer,
ISqlExecuter sqlExecuter)
{
_performanceLogger = logProvider.GetLogger("Performance");
_logger = logProvider.GetLogger(GetType().Name);
_dslModelConceptsLogger = logProvider.GetLogger("DslModelConcepts");
_dslContainer = dslContainer;
_sqlExecuter = sqlExecuter;
}
private string LogCreatedConcepts(DslContainer dslContainer, IEnumerable <IConceptInfo> macroCreatedConcepts, DslContainer.AddNewConceptsReport newConceptsReport)
{
var report = new StringBuilder();
var newUniqueIndex = new HashSet <string>(newConceptsReport.NewUniqueConcepts.Select(c => c.GetKey()));
LogConcepts(report, "Macro created", macroCreatedConcepts, first: true);
LogConcepts(report, "New unique", newConceptsReport.NewUniqueConcepts);
LogConcepts(report, "New resolved", newConceptsReport.NewlyResolvedConcepts.Where(c => newUniqueIndex.Contains(c.GetKey())));
LogConcepts(report, "Old resolved", newConceptsReport.NewlyResolvedConcepts.Where(c => !newUniqueIndex.Contains(c.GetKey())));
LogConcepts(report, "New unresolved", newConceptsReport.NewUniqueConcepts.Where(c => dslContainer.FindByKey(c.GetKey()) == null));
return(report.ToString());
}
private DslContainer Initialize(DslContainer dslContainer)
{
var swTotal = Stopwatch.StartNew();
var parsedConcepts = _dslParser.ParsedConcepts;
var swFirstAdd = Stopwatch.StartNew();
dslContainer.AddNewConceptsAndReplaceReferences(parsedConcepts);
_performanceLogger.Write(swFirstAdd, $"Initialize: First AddNewConceptsAndReplaceReferences ({dslContainer.Concepts.Count()} concepts).");
ExpandMacroConcepts(dslContainer);
CheckSemantics(dslContainer);
dslContainer.SortReferencesBeforeUsingConcept(_buildOptions.InitialConceptsSort);
ReportObsoleteConcepts(dslContainer);
_dslModelFile.SaveConcepts(dslContainer.Concepts);
_performanceLogger.Write(swTotal, $"Initialize ({dslContainer.Concepts.Count()} concepts).");
return(dslContainer);
}
private void LogDslModel(DslContainer dslContainer)
{
var sw = Stopwatch.StartNew();
// It is important to avoid generating the log data if the logger is not enabled.
var sortedConceptsLog = new Lazy <List <string> >(() => dslContainer.Concepts
.Select(c => c.GetFullDescription())
.OrderBy(log => log)
.ToList());
const int chunkSize = 10000; // Keeping the message size under NLog memory limit.
for (int start = 0; start < dslContainer.Concepts.Count(); start += chunkSize)
{
_dslModelConceptsLogger.Trace(() => string.Join("\r\n", sortedConceptsLog.Value.Skip(start).Take(chunkSize)));
}
_performanceLogger.Write(sw, "DslModel.LogDslModel.");
}
public DslModel(
IDslParser dslParser,
ILogProvider logProvider,
DslContainer dslContainer,
IIndex <Type, IEnumerable <IConceptMacro> > macros,
IEnumerable <IConceptMacro> macroPrototypes,
IEnumerable <IConceptInfo> conceptPrototypes,
IMacroOrderRepository macroOrderRepository,
IDslModelFile dslModelFile)
{
_dslParser = dslParser;
_performanceLogger = logProvider.GetLogger("Performance");
_logger = logProvider.GetLogger("DslModel");
_evaluatorsOrderLogger = logProvider.GetLogger("MacroEvaluatorsOrder");
_dslModelConceptsLogger = logProvider.GetLogger("DslModelConcepts");
_initializedDslContainer = new Lazy <DslContainer>(() => Initialize(dslContainer));
_macros = macros;
_macroTypes = macroPrototypes.Select(macro => macro.GetType());
_conceptTypes = conceptPrototypes.Select(conceptInfo => conceptInfo.GetType());
_macroOrderRepository = macroOrderRepository;
_dslModelFile = dslModelFile;
}
private void ExpandMacroConcepts(DslContainer dslContainer)
{
var swTotal = Stopwatch.StartNew();
var sw = Stopwatch.StartNew();
int iteration = 0;
var iterationCreatedConcepts = new List <IConceptInfo>();
int lastNewConceptTime = 0;
var lastNewConceptTimeByIteration = new List <int>();
var lastNewConceptTimeByMacro = new Dictionary <string, int>();
var recommendedMacroOrder = _macroOrderRepository.Load().ToDictionary(m => m.EvaluatorName, m => m.EvaluatorOrder);
var macroEvaluators = ListMacroEvaluators(recommendedMacroOrder);
var macroStopwatches = macroEvaluators.ToDictionary(macro => macro.Name, macro => new Stopwatch());
var createdTypesInIteration = new List <CreatedTypesInIteration>(dslContainer.Concepts.Count() * 5);
_performanceLogger.Write(sw, "ExpandMacroConcepts initialization ("
+ macroEvaluators.Count + " evaluators, "
+ dslContainer.Concepts.Count() + " parsed concepts resolved, "
+ dslContainer.UnresolvedConceptsCount() + " unresolved).");
do
{
iteration++;
if (iteration > MacroIterationLimit)
{
throw new DslSyntaxException(string.Format(
"Possible infinite loop detected with recursive macro concept {1}. Iteration limit ({0}) exceeded while expanding macro.",
MacroIterationLimit,
iterationCreatedConcepts.First().GetShortDescription()));
}
iterationCreatedConcepts.Clear();
_logger.Trace("Expanding macro concepts, iteration {0}.", iteration);
foreach (var macroEvaluator in macroEvaluators)
{
macroStopwatches[macroEvaluator.Name].Start();
foreach (var conceptInfo in dslContainer.FindByType(macroEvaluator.Implements, macroEvaluator.ImplementsDerivations).ToList())
{
var macroCreatedConcepts = macroEvaluator.Evaluate(conceptInfo, dslContainer);
CsUtility.Materialize(ref macroCreatedConcepts);
if (macroCreatedConcepts != null && macroCreatedConcepts.Any())
{
_logger.Trace(() => "Evaluating macro " + macroEvaluator.Name + " on " + conceptInfo.GetShortDescription() + ".");
var aiCreatedConcepts = AlternativeInitialization.InitializeNonparsableProperties(macroCreatedConcepts, _logger);
var newUniqueConcepts = dslContainer.AddNewConceptsAndReplaceReferences(aiCreatedConcepts);
newUniqueConcepts.AddRange(dslContainer.AddNewConceptsAndReplaceReferences(macroCreatedConcepts));
_logger.Trace(() => LogCreatedConcepts(dslContainer, macroCreatedConcepts, newUniqueConcepts));
iterationCreatedConcepts.AddRange(newUniqueConcepts);
// Optimization analysis:
if (newUniqueConcepts.Count > 0)
{
lastNewConceptTimeByMacro[macroEvaluator.Name] = ++lastNewConceptTime;
}
createdTypesInIteration.AddRange(newUniqueConcepts.Select(nuc =>
new CreatedTypesInIteration {
Macro = macroEvaluator.Name, Created = nuc.BaseConceptInfoType().Name, Iteration = iteration
}));
}
}
macroStopwatches[macroEvaluator.Name].Stop();
}
lastNewConceptTimeByIteration.Add(lastNewConceptTime);
_performanceLogger.Write(sw, "ExpandMacroConcepts iteration " + iteration + " ("
+ iterationCreatedConcepts.Count + " new concepts, "
+ dslContainer.UnresolvedConceptsCount() + " left unresolved).");
} while (iterationCreatedConcepts.Count > 0);
_evaluatorsOrderLogger.Trace(() => swTotal.Elapsed + "\r\n"
+ ReportLastEvaluationOrder(lastNewConceptTimeByMacro, lastNewConceptTimeByIteration));
foreach (var macroStopwatch in macroStopwatches.OrderByDescending(msw => msw.Value.Elapsed.TotalSeconds).Take(5))
{
_performanceLogger.Write(macroStopwatch.Value, () => "ExpandMacroConcepts total time for " + macroStopwatch.Key + ".");
}
_logger.Trace(() => LogCreatedTypesInIteration(createdTypesInIteration));
dslContainer.ReportErrorForUnresolvedConcepts();
SaveMacroEvaluationOrder(lastNewConceptTimeByMacro);
_performanceLogger.Write(swTotal, "ExpandMacroConcepts.");
}