コード例 #1
0
        /// <summary>
        /// Gets all callable needles for all types registered,
        /// </summary>
        /// <returns></returns>
        public aceDictionarySet <Type, String> GetAllRegistrations()
        {
            aceDictionarySet <Type, String> output = new aceDictionarySet <Type, string>();


            foreach (var p in pluginTypesByName)
            {
                output.Add(p.Value, p.Key);
            }

            foreach (var p in pluginTypesByPathName)
            {
                output.Add(p.Value, p.Key);
            }

            if (supportDirtyNaming)
            {
                foreach (var p in dirtyDictionary)
                {
                    output.Add(p.Value, p.Key);
                }
            }

            return(output);
        }
コード例 #2
0
        /// <summary>
        /// Gets the sum data set.
        /// </summary>
        /// <param name="datasets">The datasets.</param>
        /// <param name="dataSetName">Name of the data set.</param>
        /// <returns></returns>
        public static DataSet GetSumDataSet(this IEnumerable <DataSet> datasets, string dataSetName = "dataset")
        {
            DataSet dss = null;

            aceDictionarySet <string, DataTable> tableCumul = new aceDictionarySet <string, DataTable>();

            List <DataTable> fradt = new List <DataTable>();

            foreach (DataSet ds in datasets)
            {
                if (dss == null)
                {
                    dss = ds;
                }

                foreach (DataTable tb in ds.Tables)
                {
                    tableCumul.Add(tb.TableName, tb);
                }
            }

            DataSet fin = new DataSet(dataSetName);

            foreach (DataTable tb in dss.Tables)
            {
                fin.AddTable(GetAggregatedTable(tableCumul[tb.TableName], "SumOf_" + tb.TableName, dataPointAggregationAspect.overlapMultiTable)); //GetSumTable(dataSetName + "_" + tb.TableName + "_sum_" + dss.Tables.Count));
            }

            return(fin);
        }
コード例 #3
0
        /// <summary>
        /// Returns objects groupped by the same value of the specified column
        /// </summary>
        /// <typeparam name="TGroup">Value type of the column</typeparam>
        /// <param name="column">The column to use for groups</param>
        /// <param name="expression">Optional row filter expression</param>
        /// <returns></returns>
        public aceDictionarySet <TGroup, T> GetGroups <TGroup>(string column, string expression = "")
        {
            if (!table.Columns.Contains(column))
            {
                throw new ArgumentException("There is no property [" + column + "] in the table row type [" + type.Name + "]");
            }

            aceDictionarySet <TGroup, T> output = new aceDictionarySet <TGroup, T>();

            List <T> list = new List <T>();

            DataRow[] rows = null;
            if (expression.isNullOrEmpty())
            {
                rows = table.Select();
            }
            else
            {
                rows = table.Select(expression);
            }

            foreach (DataRow dr in rows)
            {
                TGroup key = (TGroup)dr[column];
                output.Add(key, (T)GetObjectFromRow(dr));
            }
            return(output);
        }
コード例 #4
0
        public String DescribeSampleDistribution(ILogBuilder modelNotes)
        {
            if (modelNotes == null)
            {
                modelNotes = new builderForLog();
            }

            var l = modelNotes.Length;

            foreach (var vc in GetCases())
            {
                modelNotes.AppendHeading("Fold: " + vc.name, 2);

                var categoryCaseList = new aceDictionarySet <String, String>();

                foreach (validationCaseCollection vcc in vc.trainingCases)
                {
                    foreach (string vccs in vcc)
                    {
                        categoryCaseList.Add(vcc.className, "[T] " + vccs);
                    }
                }

                foreach (validationCaseCollection vcc in vc.evaluationCases)
                {
                    foreach (string vccs in vcc)
                    {
                        categoryCaseList.Add(vcc.className, "[E] " + vccs);
                    }
                }

                foreach (var k in categoryCaseList.Keys)
                {
                    modelNotes.AppendHeading("Category: " + k, 3);
                    foreach (var s in categoryCaseList[k])
                    {
                        modelNotes.AppendLine(s);
                    }
                }
            }

            SampleDistributionNote = modelNotes.GetContent(l);
            SampleDistributionHash = md5.GetMd5Hash(SampleDistributionNote);
            return(SampleDistributionNote);
        }
コード例 #5
0
        public void RefineStyle(classificationReportStyleDefinition style, IEnumerable <classificationReportExpanded> reports)
        {
            local_style = style.CloneViaXML();

            if (local_style.AutoNewDataColumns || local_style.AutoHideDataColumnsWithSameData)
            {
                aceDictionarySet <String, String> knownValues = new aceDictionarySet <string, string>();

                var expData = reports.Select(x => x.data);
                foreach (var cd in expData)
                {
                    foreach (var cl in cd)
                    {
                        if (local_style.AutoNewDataColumns)
                        {
                            if (!local_style.dataColumns.Any(x => x.key == cl.key))
                            {
                                local_style.dataColumns.Add(cl);
                            }
                        }

                        if (!knownValues[cl.key].Contains(cl.value))
                        {
                            knownValues.Add(cl.key, cl.value);
                        }
                    }
                }

                if (local_style.AutoHideDataColumnsWithSameData)
                {
                    List <String> columnsToHide = new List <string>();

                    foreach (var pair in knownValues)
                    {
                        if (pair.Value.Count < 2)
                        {
                            columnsToHide.Add(pair.Key);
                        }
                        else if (pair.Value.Count == 0)
                        {
                            columnsToHide.Add(pair.Key);
                        }
                    }

                    var toRemove = local_style.dataColumns.Where(x => columnsToHide.Contains(x.key)).ToList();
                    toRemove.ForEach(x => local_style.dataColumns.Remove(x));
                }
            }
        }
コード例 #6
0
        /// <summary>
        /// Generates simple-styled graph
        /// </summary>
        /// <param name="model">The model.</param>
        /// <param name="colors">The colors: Color.LightGray, Color.WhiteSmoke, Color.OrangeRed, Color.CadetBlue, Color.Yellow, Color.SteelBlue, Color.Orchid</param>
        /// <param name="limit">The limit.</param>
        /// <returns></returns>
        public DirectedGraph GetModelGraph(pipelineModel <pipelineTaskSubjectContentToken> model, Color[] colors = null, Int32 limit = 10)
        {
            if (colors == null)
            {
                colors = new Color[] { Color.LightGray, Color.WhiteSmoke, Color.OrangeRed, Color.GreenYellow, Color.Yellow, Color.SteelBlue, Color.Orchid, Color.Yellow, Color.SteelBlue, Color.Orchid };
            }
            DirectedGraph output = new DirectedGraph();

            output.Title = model.name;

            output.Layout = imbSCI.Graph.DGML.enums.GraphLayoutEnum.Sugiyama;
            output.NeighborhoodDistance = 20;

            var st_basic       = output.Categories.AddOrGetCategory(0, nameof(pipelineNodeTypeEnum.none), "");
            var st_taskBuilder = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.taskBuilder.ToInt32(), nameof(pipelineNodeTypeEnum.taskBuilder), nameof(pipelineNodeTypeEnum.none));
            var st_distributor = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.distributor.ToInt32(), nameof(pipelineNodeTypeEnum.distributor), nameof(pipelineNodeTypeEnum.none));
            var st_transformer = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.transformer.ToInt32(), nameof(pipelineNodeTypeEnum.transformer), nameof(pipelineNodeTypeEnum.none));
            var st_bin         = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.bin.ToInt32(), nameof(pipelineNodeTypeEnum.bin), nameof(pipelineNodeTypeEnum.none));
            var st_model       = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.model.ToInt32(), nameof(pipelineNodeTypeEnum.model), nameof(pipelineNodeTypeEnum.none));

            st_basic.Stroke          = colors[0].ColorToHex();
            st_basic.StrokeThinkness = 2;
            st_basic.Background      = colors[1].ColorToHex();

            st_bin.StrokeDashArray = "5,2,5,2";

            st_taskBuilder.Background = colors[2].ColorToHex();
            st_transformer.Background = colors[3].ColorToHex();

            st_distributor.Background = colors[4].ColorToHex();

            st_bin.Background = colors[5].ColorToHex();

            st_model.Background = colors[6].ColorToHex();

            List <IPipelineNode> totalSet = new List <IPipelineNode>();
            List <IPipelineNode> nextSet  = new List <IPipelineNode>();

            nextSet.Add(model);
            Int32 i = 0;

            Node modelMainNode = null;
            Node trashBinNode  = null;
            Node exitBinNode   = null;

            while (nextSet.Any())
            {
                i++;
                List <IPipelineNode> newNextSet = new List <IPipelineNode>();
                // <---------------------------------------------------[ * ]
                foreach (IPipelineNode parent in nextSet)
                {
                    if (totalSet.Contains(parent))
                    {
                        continue;
                    }

                    if (parent == null)
                    {
                        continue;
                    }
                    parent.SetLabel();

                    Node parentNode = output.Nodes.AddNode(parent.path, parent.Label);

                    if (parentNode == null)
                    {
                        continue;
                    }

                    // <-------------------------------------------------------------------------------  //
                    if (parent.parent != null)
                    {
                        ordinalSet.Add(parent.parent as IPipelineNode, parent);                         //
                    }
                    // <-----------------------------------------------------------------------------  //

                    if (!nodeRegistry.ContainsKey(parent))
                    {
                        nodeRegistry.Add(parent, parentNode);
                    }

                    SetCategory(parentNode, parent);

                    if (parent is IPipelineModel pipeModel)
                    {
                        trashBinNode          = output.Nodes.AddNode(pipeModel.trashBin.path, pipeModel.trashBin.Label);
                        exitBinNode           = output.Nodes.AddNode(pipeModel.exitBin.path, pipeModel.exitBin.Label);
                        trashBinNode.Category = pipelineNodeTypeEnum.bin.ToInt32().ToString();
                        exitBinNode.Category  = pipelineNodeTypeEnum.bin.ToInt32().ToString();

                        if (!nodeRegistry.ContainsKey(pipeModel.trashBin))
                        {
                            nodeRegistry.Add(pipeModel.trashBin, trashBinNode);
                        }

                        if (!nodeRegistry.ContainsKey(pipeModel.exitBin))
                        {
                            nodeRegistry.Add(pipeModel.exitBin, exitBinNode);
                        }

                        //var trashNode = output.Nodes.AddNode(.path, pipeModel.trashBin.Label);

                        modelMainNode = parentNode;

                        //  AddAndLink(output, parentNode, pipeModel.exitBin, newNextSet, Color.Green).Label = "Output";
                    }

                    if (parent is pipelineContentTokenLevelDistribution distNode)
                    {
                        if (distNode != null)
                        {
                            AddAndLink(output, parentNode, distNode?.repoPipeline, newNextSet, Color.Violet, "Repository");
                            AddAndLink(output, parentNode, distNode?.sitePipeline, newNextSet, Color.Violet, "Site");
                            AddAndLink(output, parentNode, distNode?.pagePipeline, newNextSet, Color.Violet, "Page");
                            AddAndLink(output, parentNode, distNode?.blockPipeline, newNextSet, Color.Violet, "Block");
                            AddAndLink(output, parentNode, distNode?.streamPipeline, newNextSet, Color.Violet, "Stream");
                            AddAndLink(output, parentNode, distNode?.chunkPipeline, newNextSet, Color.Violet, "Chunk");
                            AddAndLink(output, parentNode, distNode?.tokenPipeline, newNextSet, Color.Violet, "Token");
                        }
                    }
                    else if (parent.nodeType == pipelineNodeTypeEnum.distributor)
                    {
                        parentNode.Label = GetLabel(parent);
                    }
                    else
                    {
                        var lNext = AddAndLink(output, parentNode, parent.next, newNextSet, Color.SteelBlue, "Next");
                        if (parent.next == model.exitBin)
                        {
                            if (lNext != null)
                            {
                                lNext.StrokeDashArray = "2,3,2,3";
                                lNext.Label           = "Done";
                            }
                        }

                        var lForward = AddAndLink(output, parentNode, parent.forward, newNextSet, Color.OrangeRed, "Forward");
                    }

                    if (parent.nodeType == pipelineNodeTypeEnum.distributor)
                    {
                        var tl = output.Links.AddLink(parentNode, trashBinNode, "Removed");
                        tl.Stroke          = Color.Gray.ColorToHex();
                        tl.StrokeDashArray = "2,2,5,2,5";
                        tl.StrokeThinkness = 8;
                    }

                    if (parent.GetType().Name.Contains("TaskBuilder"))
                    {
                        if (modelMainNode != null)
                        {
                            var ntl = output.Links.AddLink(parentNode, modelMainNode, "New Tasks");
                            ntl.Stroke          = Color.MediumVioletRed.ColorToHex();
                            ntl.StrokeDashArray = "2,2,5,2,5";
                            ntl.StrokeThinkness = 8;
                        }
                    }

                    if (parent.parent != null)
                    {
                        IPipelineNode parentFolder = parent.parent as IPipelineNode;

                        var l = new Link(parentFolder.path, parentNode.Id, true);

                        if (parentFolder.next == parent)
                        {
                            l.Stroke          = Color.SteelBlue.ColorToHex();
                            l.Label           = "No";
                            l.StrokeThinkness = 4;
                            output.Links.Add(l);
                        }
                        else if (parentFolder.forward == parent)
                        {
                            l.Stroke          = Color.OrangeRed.ColorToHex();
                            l.Label           = "Execute";
                            l.StrokeThinkness = 6;
                            output.Links.Add(l);
                        }
                        else
                        {
                            l.Stroke          = Color.DarkGray.ColorToHex();
                            l.Label           = "Owner";
                            l.StrokeDashArray = "2,5,2,5";
                            l.StrokeThinkness = 2;
                        }
                        // output.Links.Add(l);
                    }

                    foreach (var pair in parent)
                    {
                        if (!newNextSet.Contains(pair))
                        {
                            newNextSet.Add(pair as IPipelineNode);
                            totalSet.Add(pair as IPipelineNode);
                        }
                    }
                }

                if (i > limit)
                {
                    break;
                }
                nextSet = newNextSet;
            }
            // <---------------------------------------------------[ * ]

            // <---------------------------------------------------[ * ]
            foreach (var pair in ordinalSet)
            {
                IPipelineNode firstPipelineNode = null;
                Node          firstNode         = null;

                IPipelineNode currentPipelineNode = null;
                Node          currentNode         = null;

                foreach (IPipelineNode p in pair.Value)
                {
                    if (firstPipelineNode == null)
                    {
                        firstPipelineNode = p;
                        firstNode         = nodeRegistry[firstPipelineNode];
                    }
                    else
                    {
                        //var ld = output.Links.AddLink(firstNode, nodeRegistry[p], "Next");
                        //ld.Stroke = Color.SteelBlue.ColorToHex();
                        //ld.StrokeDashArray = "2,5,2,5";
                        //ld.StrokeThinkness = 4;
                        firstNode = nodeRegistry[p];
                    }
                    currentNode         = nodeRegistry[p];
                    currentPipelineNode = p;
                }

                if (pair.Key.level > 1)
                {
                    var l = output.Links.AddLink(currentNode, nodeRegistry[model.exitBin], "Done");
                    l.Stroke = Color.SteelBlue.ColorToHex();
                    //l.StrokeDashArray = "";
                    l.StrokeThinkness = 4;
                }
            }
            // <---------------------------------------------------[ * ]

            // <---------------------------------------------------[ * ]
            var allEnums = Enum.GetValues(typeof(pipelineNodeTypeEnum));

            var LEGEND = output.Nodes.AddNode("LEGEND");

            foreach (pipelineNodeTypeEnum en in allEnums)
            {
                var n = output.Nodes.AddNode("LEG" + en.ToString(), en.ToString().imbTitleCamelOperation(true));
                n.Category = en.ToInt32().ToString();
                var l = output.Links.AddLink(LEGEND, n, "");
                l.StrokeDashArray = "5,5,5,5";
            }
            // <---------------------------------------------------[ * ]

            //var st_basic = output.Categories.AddOrGetCategory(0, nameof(pipelineNodeTypeEnum.none), "");
            //var st_taskBuilder = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.taskBuilder.ToInt32(), nameof(pipelineNodeTypeEnum.taskBuilder), nameof(pipelineNodeTypeEnum.none));
            //var st_distributor = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.distributor.ToInt32(), nameof(pipelineNodeTypeEnum.distributor), nameof(pipelineNodeTypeEnum.none));
            //var st_transformer = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.transformer.ToInt32(), nameof(pipelineNodeTypeEnum.transformer), nameof(pipelineNodeTypeEnum.none));
            //var st_bin = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.bin.ToInt32(), nameof(pipelineNodeTypeEnum.bin), nameof(pipelineNodeTypeEnum.none));
            //var st_model = output.Categories.AddOrGetCategory(pipelineNodeTypeEnum.model.ToInt32(), nameof(pipelineNodeTypeEnum.model), nameof(pipelineNodeTypeEnum.none));

            return(output);
        }
コード例 #7
0
        public static DataTable GetParallelAggregates <T>(this List <IEnumerable <T> > sources, string column_snap, string column_prefix, dataPointAggregationType column_sideAggregates, params string[] column_toInclude)
        {
            settingsEntriesForObject sEO        = new settingsEntriesForObject(typeof(T));
            settingsPropertyEntry    sPE_snap   = sEO.spes[column_snap];
            settingsPropertyEntry    sPE_prefix = sEO.spes[column_prefix];

            List <settingsPropertyEntry> sPE_toInclude = new List <settingsPropertyEntry>();

            foreach (string toInclude in column_toInclude)
            {
                sPE_toInclude.Add(sEO.spes[toInclude]);
            }

            List <dataPointAggregationType> side_aggregates = column_sideAggregates.getEnumListFromFlags <dataPointAggregationType>();

            Dictionary <dataPointAggregationType, settingsPropertyEntry> sPE_sideAggregates = new Dictionary <dataPointAggregationType, settingsPropertyEntry>();

            Dictionary <settingsPropertyEntry, dataPointAggregationType> sPE_sideAggregatesContra = new Dictionary <settingsPropertyEntry, dataPointAggregationType>();

            foreach (settingsPropertyEntry toInclude in sPE_toInclude)
            {
                foreach (dataPointAggregationType sideType in side_aggregates)
                {
                    settingsPropertyEntry sPE = new settingsPropertyEntry(toInclude.pi);
                    sPE.type = typeof(double);
                    sPE.name = sPE.name + "_" + sideType.ToString();
                    sPE_sideAggregates.Add(sideType, sPE);
                    sPE_sideAggregatesContra.Add(sPE, sideType);
                }
            }

            // <---------------------------- preparing data

            Dictionary <string, IEnumerable <T> > dataByPrefix = new Dictionary <string, IEnumerable <T> >();
            int c = 0;

            foreach (IEnumerable <T> s in sources)
            {
                T firstItem = s.FirstOrDefault <T>();
                if (firstItem != null)
                {
                    string prefix = firstItem.imbGetPropertySafe(sPE_prefix.pi).toStringSafe(c.ToString("D3"));
                    dataByPrefix.Add(prefix, s);
                }
                c++;
            }

            // <----- DataColumn Index

            aceDictionarySet <string, DataColumn> columnsByPrefix = new aceDictionarySet <string, DataColumn>();

            aceDictionarySet <string, DataColumn> columnsSideAggregationByPrefix = new aceDictionarySet <string, DataColumn>();

            // <------------------------- building Shema
            DataTable output = new DataTable();

            output.TableName = "ParallelAggregate_by_" + column_snap;

            DataColumn            col_recordID = output.Add("ID", "Row ordinal number", "ID", typeof(int), dataPointImportance.normal, "D3").SetUnit("#");
            settingsPropertyEntry sPE_recID    = col_recordID.GetSPE();

            DataColumn col_snap = output.Add(sPE_snap);

            aceDictionary2D <settingsPropertyEntry, dataPointAggregationType, DataColumn> columnsByAggregationType = new aceDictionary2D <settingsPropertyEntry, dataPointAggregationType, DataColumn>();
            aceDictionarySet <settingsPropertyEntry, DataColumn> columnsBySource = new aceDictionarySet <settingsPropertyEntry, DataColumn>();

            foreach (settingsPropertyEntry toInclude in sPE_toInclude)
            {
                foreach (var pair in dataByPrefix)
                {
                    DataColumn nColumn = output.Add(toInclude);
                    nColumn.ColumnName = pair.Key + "_" + nColumn.ColumnName;
                    nColumn.SetGroup(pair.Key);

                    columnsByPrefix.Add(pair.Key, nColumn);

                    columnsBySource.Add(toInclude, nColumn);
                }

                foreach (var pair2 in sPE_sideAggregatesContra)
                {
                    DataColumn nColumn2 = output.Add(toInclude);
                    nColumn2.SetGroup("Aggregate");

                    //                    columnsSideAggregationByPrefix.Add(pair.Key, nColumn);
                }
            }

            // <----------------------------------------------------------- collecting rows

            aceDictionary2D <string, settingsPropertyEntry, object> dataRowBySnapValue = new aceDictionary2D <string, settingsPropertyEntry, object>();

            int riMax = 0;

            foreach (string prefix in dataByPrefix.Keys)
            {
                IEnumerable <T> s  = dataByPrefix[prefix];
                int             ri = 0;
                foreach (T si in s)
                {
                    ri++;
                    string snapValue = si.imbGetPropertySafe(sPE_snap.pi).toStringSafe();

                    dataRowBySnapValue[snapValue, sPE_snap]  = snapValue;
                    dataRowBySnapValue[snapValue, sPE_recID] = ri;

                    foreach (settingsPropertyEntry toInclude in sPE_toInclude)
                    {
                        foreach (var pair in columnsByPrefix[prefix])
                        {
                            var spe = dataColumnRenderingSetup.GetSPE(pair);

                            dataRowBySnapValue[snapValue, spe] = si.imbGetPropertySafe(spe.pi);
                        }
                    }
                    riMax = Math.Max(ri, riMax);
                }
            }

            foreach (string prefix in dataByPrefix.Keys)
            {
            }

            //List<Double> data = new List<Double>();
            //foreach (var pair2 in columnsSideAggregationByPrefix[prefix])
            //{
            //    var spe2 = pair.GetSPE();

            //    dataRowBySnapValue[snapValue, spe2] = si.imbGetPropertySafe(spe2.pi);
            //}

            return(output);
        }
コード例 #8
0
        /// <summary>
        /// Populates the graph report
        /// </summary>
        /// <param name="graph">The graph.</param>
        public void Deploy(freeGraph graph)
        {
            name = graph.name;

            description = "Analysis of graph [" + graph.name + "] structure and other key metrics: " + graph.description;

            List <Double> ws = new List <double>();
            //instanceCountCollection<freeGraphNodeBase> linkPerNodeFrequency = new instanceCountCollection<freeGraphNodeBase>();
            aceDictionarySet <Int32, freeGraphNodeBase> nodesByNumberOfLinks = new aceDictionarySet <int, freeGraphNodeBase>();

            foreach (var node in graph.nodes)
            {
                ws.Add(node.weight);

                Int32 lc = graph.CountLinks(node.name, true, true);
                nodesByNumberOfLinks.Add(lc, node);
            }

            if (!ws.Any())
            {
                EmptyGraph = true;
                return;
            }

            TotalWeight = ws.Sum();
            AvgWeight   = ws.Average();
            StdWeight   = ws.GetStdDeviation(false);

            NodeCount = graph.nodes.Count;
            LinkRatio = graph.links.GetRatio(graph.nodes);

            List <String> processed       = new List <String>();
            List <Int32>  linkFrequencies = nodesByNumberOfLinks.Keys.OrderByDescending(x => x).ToList();

            Int32 maxLinkFreq = linkFrequencies.Max();

            foreach (Int32 freq in linkFrequencies)
            {
                List <freeGraphNodeBase> nextSet = nodesByNumberOfLinks[freq].ToList();
                foreach (freeGraphNodeBase n in nextSet)
                {
                    if (!processed.Contains(n.name))
                    {
                        var             result = graph.GetLinkedNodes(new String[] { n.name }, 100, true, true, false);
                        freeGraphIsland island = new freeGraphIsland();
                        island.Add(result);

                        if (island.type != freeGraphIslandType.none)
                        {
                            islands.Add(island);
                            processed.AddRange(island.nodes);
                        }

                        //result.ForEach(x => processed.Add(x.name));
                    }
                }
            }

            List <freeGraphIsland> _binodal   = new List <freeGraphIsland>();
            List <freeGraphIsland> _trinodal  = new List <freeGraphIsland>();
            List <freeGraphIsland> _polinodal = new List <freeGraphIsland>();

            freeGraphIsland continent = null;

            if (islands.Any())
            {
                Int32 max = 0;
                continent = islands.First();
                foreach (var island in islands)
                {
                    Int32 nc = island.nodes.Count;
                    if (nc == 2)
                    {
                        Binodal += nc;
                        BinodalN++;
                    }
                    if (nc == 3)
                    {
                        Trinodal += nc;
                        TrinodalN++;
                    }
                    if (nc > 3)
                    {
                        Polinodal += nc;
                        PolinodalN++;
                    }
                    if (max < nc)
                    {
                        max       = nc;
                        continent = island;
                    }
                }
            }

            AllIslands = islands.Count;

            PolinodalRatio = Polinodal.GetRatio(graph.nodes.Count);

            if (continent != null)
            {
                var contNodes = graph.nodes.Where(x => continent.nodes.Contains(x.name)).ToList();

                //var contNodes = ;

                ContinentMass = continent.nodes.Count;
                Double cw = contNodes.Sum(x => x.weight);
                ContinentWeightRate = cw.GetRatio(TotalWeight);
                ContinentMassRate   = continent.nodes.Count.GetRatio(graph.nodes.Count);

                ContinentSize = graph.PingGraphSize(nodesByNumberOfLinks[maxLinkFreq], true, freeGraphPingType.maximumPingLength);
            }

            MainIsland = continent;
        }
コード例 #9
0
        /// <summary>Performs post-processing of data collected by the workload plugin</summary>
        /// <remarks><para>Loads all saved DataTables, groups rows in averages for each measure group and creates summary table with all experiments</para></remarks>
        /// <param name="searchPattern">pattern used to select input files</param>
        /// <param name="groupColumn">column name used for row grouping</param>
        /// <param name="overviewColumns">columns to include in overview table</param>
        /// <seealso cref="aceOperationSetExecutorBase"/>
        public void aceOperation_runWorkloadData(
            [Description("pattern used to select input files")] string searchPattern = "results*.xml",
            [Description("column name used for row grouping")] string groupColumn    = "measureGroup")
        // [Description("columns to include in overview table")] String overviewColumns = "DataLoad,CrawlerIterations,ContentPages,dlcMaximum")
        {
            aceOperation_selectFiles(searchPattern, "index\\workload", true);

            folder = folder["index\\workload"];

            List <DataTable> tables = new List <DataTable>();

            dataPointAggregationType aggType = dataPointAggregationType.avg;

            int ci = 1;
            int c  = selectedFiles.Count();

            output.log("[" + c + "] DataTable in the cue.");


            List <DataTable> allTables = new List <DataTable>();
            DataSet          dSet      = new DataSet();


            aceDictionarySet <string, DataTable> byCrawler = new aceDictionarySet <string, DataTable>();
            aceDictionarySet <string, DataTableForStatistics> byCrawlerRT = new aceDictionarySet <string, DataTableForStatistics>();

            DataTableForStatistics rt = null;

            foreach (FileInfo fi in selectedFiles)
            {
                try
                {
                    objectTable <reportPlugIn_workloadEntry> workloadEntry = new objectTable <reportPlugIn_workloadEntry>(fi.FullName, true, "EntryID", "");

                    objectTable <reportPlugIn_workloadEntry> workloadGrouped = new objectTable <reportPlugIn_workloadEntry>("EntryID", "aggregated");

                    aceDictionarySet <int, reportPlugIn_workloadEntry> workloadGroups = workloadEntry.GetGroups <int>(groupColumn, "terminationWarning = 0");

                    collectionAggregationResultSet <reportPlugIn_workloadEntry> aggregateSet = new collectionAggregationResultSet <reportPlugIn_workloadEntry>();



                    foreach (var set in workloadGroups)
                    {
                        collectionAggregationResult <reportPlugIn_workloadEntry> aggregates = null;
                        aggregates = set.Value.GetAggregates(aggType);

                        var aggregate = aggregates[aggType];
                        aggregate.measureGroup = set.Key;
                        aggregate.EntryID      = set.Key.ToString("D5") + "_" + aggType.ToString();
                        workloadGrouped.AddOrUpdate(aggregate);
                        aggregateSet.Add(aggregate.EntryID + "_" + fi.Name, aggregates);
                    }

                    string filename = (fi.Name + "_" + groupColumn + "_" + aggType.ToString()).getFilename();

                    string n = reportPlugIn_workload_state.ExtractEntryID(aggregateSet.lastItem.EntryID) + dSet.Tables.Count.ToString("D2");

                    DataTable dt = workloadGrouped.GetDataTable(dSet, n);
                    dt.SetDescription("Collection of [" + aggregateSet.recordType.Name + "] records grouped by [" + groupColumn + "]");
                    dt.SetAggregationAspect(dataPointAggregationAspect.subSetOfRows);
                    dt.SetAggregationOriginCount(aggregateSet.Count);
                    dt.SetAdditionalInfoEntry("Aggregation Type:", aggType);
                    dt.SetAdditionalInfoEntry("Data source file:", fi.Name);

                    dt.SetAdditionalInfoEntries("Last", aggregateSet.lastItem, typeof(string));

                    dt.SetTitle(n);

                    byCrawler.Add(aggregateSet.firstItem.Crawler, dt);

                    // dt.TableName = n;
                    //   dSet.AddTable(dt);


                    rt = dt.GetReportAndSave(folder, imbWEMManager.authorNotation, n.getFilename(), true);
                    byCrawlerRT.Add(aggregateSet.firstItem.Crawler, rt);
                    response.AppendLine("[" + ci + " / " + c + "] DataTable [" + fi.Name + "] had [" + workloadGroups.Keys.Count() + "] groups. Result saved as: " + filename);
                    ci++;
                } catch (Exception ex)
                {
                    output.log("[" + ci + " / " + c + "] DataTable [" + fi.FullName + "] failed.");
                    output.log(ex.Message);
                }
            }



            output.log("[" + c + "] DataTable processed.");

            dSet.serializeDataSet("workload_all", folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);

            foreach (string key in byCrawler.Keys)
            {
                string  filename = key.getFilename();
                DataSet sd       = new DataSet(key);
                foreach (DataTable dti in byCrawler[key])
                {
                    sd.AddTable(dti.Copy());
                }

                sd.AddTable(byCrawlerRT[key].First().RenderLegend());
                sd.serializeDataSet(filename, folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);
            }
        }
コード例 #10
0
        /// <summary>Exporting domain list according to criteria specified</summary>
        /// <remarks><para>It will select domains using existing data. If index name not specified it will scan index repository and ask user to pick one</para></remarks>
        /// <param name="indexName">name of the index to harvest sample from - IndexID</param>
        /// <param name="minPages">required min. number of crawled/indexed pages in the doman--</param>
        /// <param name="minRelevant">required min. number of relevant pages in the index for the domain</param>
        /// <seealso cref="aceOperationSetExecutorBase"/>
        public void aceOperation_runIndexData(
            [Description("name of the index to harvest sample from - IndexID")] string indexName                = "MainIndex",
            [Description("required min. number of crawled/indexed pages in the doman--")] int minPages          = 30,
            [Description("required min. number of relevant pages in the index for the domain")] int minRelevant = 10)
        {
            if ((indexName == "*") || indexName.isNullOrEmpty())
            {
                List <string> indexList = imbWEMManager.index.GetIndexList();
                indexList.Add("*");
                aceTerminalInput.askForOption("Choose index to work with - or confirm * to load all indexes:", "*", indexList);
            }

            indexDatabaseStandalone indexDb = new indexDatabaseStandalone(indexName);



            imbWEMManager.index.OpenIndex(indexName, "plugin_dataLoader");

            imbWEMManager.index.pageIndexTable.ReadOnlyMode   = true;
            imbWEMManager.index.domainIndexTable.ReadOnlyMode = true;
            List <indexDomain> d_list = new List <indexDomain>();

            List <indexPage> pages = imbWEMManager.index.pageIndexTable.GetPagesAndDomains(indexPageEvaluationEntryState.inTheIndex, out d_list);

            aceDictionarySet <indexDomain, indexPage> dict = new aceDictionarySet <indexDomain, indexPage>();
            List <string>      list    = new List <string>();
            List <indexDomain> domains = new List <indexDomain>();

            foreach (indexDomain domain in d_list)
            {
                List <indexPage> pl = Enumerable.Where(pages, x => x.url.Contains(domain.domain)).ToList();
                dict.Add(domain, pl);
                int prc = 0;
                if (pl.Count() > minPages)
                {
                    foreach (indexPage ip in pl)
                    {
                        if (ip.relevancyText == "isRelevant")
                        {
                            prc++;
                        }
                        if (prc > minRelevant)
                        {
                            output.AppendLine($" {domain.domain} P[_{pl.Count()}_] Pr[_{prc}_] --> accepted, stop counting");
                            //  domains.Add(domain);
                            list.Add(domain.domain);
                            break;
                        }
                    }
                }
            }

            //  var domains = imbWEMManager.index.domainIndexTable.GetWhere(nameof(indexDomain.relevantPages) + " > " + minRelevant);
            // domains = domains.Where(x => ((x.relevantPages + x.notRelevantPages) > minPages)).ToList();
            string sampleName = indexName.add("Pa" + minPages + "Pr" + minRelevant, "_").add("txt", ".");

            domains.ForEach(x => list.Add(x.url));


            objectTable <indexDomain> dTable = new objectTable <indexDomain>("url", sampleName);

            domains.ForEach(x => dTable.AddOrUpdate(x));

            dTable.GetDataTable(null, sampleName).GetReportAndSave(folder, imbWEMManager.authorNotation, sampleName, true);

            folder = imbWEMManager.index.folder;

            string p = folder.pathFor(sampleName);

            list.saveContentOnFilePath(p);

            output.log("Exported sample saved to: " + p);
        }
コード例 #11
0
        protected void runModel(experimentExecutionContext context, IWebFVExtractor model)
        {
            imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, model.name);
            Int32 crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
            aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection> casesByClasses = new aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection>();
            DSCCReportSet kFoldReport = new DSCCReportSet(model);
            var           valCol      = context.validationCollections[model.name];

            List <DocumentSetCaseCollectionSet> modelCaseResults = new List <DocumentSetCaseCollectionSet>();

            crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
            while (crashRetries > 0)
            {
                try
                {
                    experimentNotes modelNotes = new experimentNotes(valCol.folder, "Fold-level experiment settings description notes");
                    modelNotes.AppendLine("# Notes on Feature Vector Extractor: " + model.name);

                    var nts = model.DescribeSelf();
                    nts.ForEach(x => modelNotes.AppendLine(x));



                    context.logger.log("Executing k-fold cases with model [" + model.name + "]");



                    valCol.DescribeSampleDistribution(modelNotes);

                    context.mainReport.valColVsModelVsSampleHash.Add("[" + model.name + "]".toWidthExact(20) + " [sample distribution hash: " + valCol.SampleDistributionHash + "]");

                    modelNotes.SaveNote();

                    ParallelOptions ops = new ParallelOptions();
                    ops.MaxDegreeOfParallelism = context.tools.operation.ParallelThreads;

                    Parallel.ForEach <kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
                    {
                        model.DoFVEAndTraining(valCase, context.tools, context.logger); // <---------------------------------------------------------------------------------------   BUILDING FVE

                        DocumentSetCaseCollectionSet results = model.DoClassification(valCase, context.tools, context.logger);

                        if (!results.Any())
                        {
                            throw new aceScienceException("DoClassification for [" + model.name + "] returned no results!", null, model, "DoClassification " + model.name + " failed!", context);
                        }

                        foreach (var pair in results)
                        {
                            DocumentSetCaseCollection cls = pair.Value;
                            casesByClasses.Add(cls.setClass, cls);
                        }

                        valCase.evaluationResults = results;

                        if (context.tools.DoResultReporting)
                        {
                            context.logger.log("producing reports on k-Fold case [" + valCase.name + "]");
                            DSCCReports r = results.GetReports();

                            var sumMeans = r.GetAverageTable(context); //.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name);
                            sumMeans.SetDescription("FVE report, aggregated for all categories - for fold [" + valCase.name + "]");


                            sumMeans.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name, true, context.tools.operation.doReportsInParalell);

                            var fveAndCase = r.GetFullValidationTable(context);
                            fveAndCase.SetDescription("Per-category aggregate statistics, for each classifier, within fold [" + valCase.name + "], used for macro-averaging");
                            fveAndCase.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_extrainfo_" + valCase.name, true, context.tools.operation.doReportsInParalell);

                            var fullCaseReport = results.GetReportOnAllCases();


                            fullCaseReport.GetReportAndSave(valCase.folder, appManager.AppInfo, "FullReport_" + valCase.name, true, context.tools.operation.doReportsInParalell);

                            kFoldReport.Add(valCase, r);
                        }

                        context.logger.log("k-Fold case [" + valCase.name + "] completed");

                        context.notes.log("- - Experiment sequence for [" + valCase.name + "] fold completed");
                        if (context.tools.operation.doSaveKnowledgeForClasses)
                        {
                            valCase.knowledgeLibrary.SaveKnowledgeInstancesForClasses(valCase, context.logger);
                        }
                    });

                    foreach (var fold in valCol.GetCases()) //  Parallel.ForEach<kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
                    {
                        modelCaseResults.Add(fold.evaluationResults);
                    }

                    crashRetries = 0;
                }
                catch (Exception ex)
                {
                    crashRetries--;
                    context.errorNotes.LogException("FVE Model crashed -- retries left [" + crashRetries + "] --- ", ex, model.name);
                    context.logger.log(":::: REPEATING the model [" + model.name + "] ::: CRASHED [" + ex.Message + "] ::: RETRIES [" + crashRetries + "]");
                    imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(1200, 1000, 1);
                    imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(2400, 1000, 1);
                    imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "RETRIES[" + crashRetries + "]");
                }
            }


            imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "Reporting");


            valCol.knowledgeLibrary.SaveCaseKnowledgeInstances(context.logger);

            // DocumentSetCaseCollection second = null;
            if (modelCaseResults.Any())
            {
                featureExtractionMetrics modelMetrics = new featureExtractionMetrics(model.name, "All");
                DataTableTypeExtended <featureExtractionMetrics> modelVsCategoryMetrics = new DataTableTypeExtended <featureExtractionMetrics>(model.name, "Model metrics per category");


                // <-------------------------------------- CATEGORIES REPORT ----------------------------------------------

                DataTable allTable = modelCaseResults.First()[0].GetReportTable(false, false).GetClonedShema <DataTable>();; //valCol.GetCases().First().evaluationResults[0].GetReportTable(false, false);


                rangeFinderForDataTable ranger = new rangeFinderForDataTable(allTable, "name");
                ranger.columnsToSignIn.Add("Case");

                foreach (KeyValuePair <IDocumentSetClass, aceConcurrentBag <DocumentSetCaseCollection> > pair in casesByClasses)
                {
                    DocumentSetCaseCollection first = null;
                    DataTable repTable = null;

                    ranger.prepareForNextAggregationBlock(allTable, "name");

                    foreach (DocumentSetCaseCollection cn in pair.Value)
                    {
                        foreach (var cni in cn)
                        {
                            if (cni != null)
                            {
                                cn.BuildRow(cni, allTable, false);
                            }
                        }
                    }

                    ranger.AddRangeRows(pair.Key.name, allTable, true, imbSCI.Core.math.aggregation.dataPointAggregationType.avg | imbSCI.Core.math.aggregation.dataPointAggregationType.stdev);

                    var categoryMetrics = new featureExtractionMetrics(model.name, pair.Key.name);
                    categoryMetrics.SetValues(ranger);

                    modelVsCategoryMetrics.AddRow(categoryMetrics);
                    modelMetrics.AddValues(categoryMetrics);

                    categoryMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_" + categoryMetrics.Name + ".xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "FV and Category sample metrics, serialized object"));
                    //context.notes.log("- - Creating report for category [" + pair.Key.name + "] completed");
                    //repTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_category_" + pair.Key.name);
                }

                modelMetrics.DivideValues(casesByClasses.Count);
                modelMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_metrics.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Cross-categories macroaveraged metrics of the FVE model [" + model.name + "]"));

                modelVsCategoryMetrics.AddRow(modelMetrics);
                modelVsCategoryMetrics.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, "Name", modelMetrics.Name);
                modelVsCategoryMetrics.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_metrics", true, true);

                context.mainReport.AddModelMetrics(modelMetrics);


                context.notes.log("- Creating report for all categories [" + model.name + "] ");
                allTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_categories", true, context.tools.operation.doReportsInParalell);
            }



            kFoldReport.MakeReports(context, valCol.folder);
            context.mainReport.AddBestPerformer(kFoldReport.GetTopClassifierReport(), kFoldReport.meanClassifierReport, model);

            // <---------------- creation of complete report

            context.notes.log("- Experiment sequence with Feature Vector Extractor [" + model.name + "] completed");
            context.notes.SaveNote();

            // <------------- END OF THE MODEL -------------------------------------------------------------------------------------------------
        }
コード例 #12
0
        /// <summary>
        /// Builds the type of the tree node for.
        /// </summary>
        /// <param name="type">The type.</param>
        /// <param name="source">The source.</param>
        /// <param name="host">The host.</param>
        /// <returns></returns>
        internal static commandTreeDescription BuildTreeNodeForType(Type type, IAceOperationSetExecutor source, commandTreeDescription parent, String nameOverride = "", commandTreeNodeLevel level = commandTreeNodeLevel.type)
        {
            settingsMemberInfoEntry typeInfo = new settingsMemberInfoEntry(type);

            commandTreeDescription output = parent.Add(nameOverride.or(typeInfo.name, typeInfo.displayName, type.Name));

            commandTree host = parent.root as commandTree;

            output.description = typeInfo.description;
            output.nodeLevel   = level;
            output.helpLines.Add(typeInfo.info_helpTitle);
            output.helpLines.Add(typeInfo.info_helpTips);
            output.helpLines.AddRange(typeInfo.additionalInfo);


            var methods = type.GetMethods(System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.DeclaredOnly);

            aceDictionarySet <String, commandTreeDescription> groups = new aceDictionarySet <string, commandTreeDescription>();

            aceDictionarySet <String, commandTreeDescription> group_nodes = new aceDictionarySet <string, commandTreeDescription>();


            foreach (MemberInfo __m in methods)
            {
                if (__m.Name.StartsWith(aceMenuItemMeta.METHOD_PREFIX))
                {
                    commandTreeDescription desc = new commandTreeDescription();

                    desc.setByMemberInfo(__m);
                    desc.nodeLevel = commandTreeNodeLevel.group;
                    groups.Add(desc.category, desc);
                }
            }

            foreach (String group in groups.Keys.OrderBy(x => x))
            {
                var ordered = groups[group].OrderBy(x => x.name);

                commandTreeDescription gdesc = parent.Add(group);

                gdesc.nodeLevel = commandTreeNodeLevel.group;

                foreach (var cdesc in ordered)
                {
                    cdesc.nodeLevel = commandTreeNodeLevel.command;
                    gdesc.Add(cdesc, cdesc.name);

                    host.flatAccess.Add(cdesc.path, cdesc);
                }
            }

            var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.DeclaredOnly);

            foreach (PropertyInfo pi in properties)
            {
                if (pi.DeclaringType.Name == "aceCommandConsole")
                {
                    continue;
                }
                if (pi.DeclaringType.Name == "aceAdvancedConsole")
                {
                    continue;
                }

                if (pi.PropertyType.GetInterfaces().Contains(typeof(IAceOperationSetExecutor)))
                {
                    var plugin_instance = source.imbGetPropertySafe(pi) as IAceOperationSetExecutor;
                    var plugin_node     = BuildTreeNodeForType(pi.PropertyType, source, parent, pi.Name, commandTreeNodeLevel.plugin);
                    plugin_node.setByMemberInfo(pi);

                    host.plugins.Add(plugin_node.path, plugin_instance);
                }
                else if (pi.PropertyType.IsValueType || pi.PropertyType.isTextOrNumber() || pi.GetCustomAttributes(false).Any())
                {
                    if (pi.CanWrite)
                    {
                        var prop = parent.Add(pi.Name);

                        prop.setByMemberInfo(pi);
                        prop.nodeLevel = commandTreeNodeLevel.parameter;
                        host.properties.Add(prop.path, pi);
                    }
                }
                else if (pi.PropertyType.IsClass || pi.GetCustomAttributes(false).Any())
                {
                    if (!pi.PropertyType.IsGenericType)
                    {
                        var prop = parent.Add(pi.Name);
                        prop.setByMemberInfo(pi);
                        prop.nodeLevel = commandTreeNodeLevel.module;
                        host.modules.Add(prop.path, pi);
                    }
                }
            }

            return(output);
        }
コード例 #13
0
        public override void eventCrawlJobFinished(crawlerDomainTaskMachine __machine, modelSpiderTestRecord tRecord)
        {
            crawlerReportFolder = __machine.reporter.folder;

            var    wRecords   = tRecord.GetChildRecords();
            string fileprefix = tRecord.instance.name.getFilename();

            List <DataTable> iterationTimelines = new List <DataTable>();

            foreach (var wRecord in wRecords)
            {
                iterationTimelines.Add(wRecord.iterationTableRecord.GetDataTable());
            }
            int dlc_c = 0;

            if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
            {
                tRecord.frontierDLCDataTables[moduleIterationRecordSummary.fra_overview].GetAggregatedTable("fra_overview").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_overview".add(fileprefix, "_"), true);

                tRecord.frontierDLCDataTables[moduleIterationRecordSummary.all].GetAggregatedTable("fra_modules_all").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_modules_all".add(fileprefix, "_"), true);


                if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.language].Any())
                {
                    tRecord.frontierDLCDataTables[moduleIterationRecordSummary.language].GetAggregatedTable("fra_module_language").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_language_".add(fileprefix, "_"), true);
                }
                if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.structure].Any())
                {
                    tRecord.frontierDLCDataTables[moduleIterationRecordSummary.structure].GetAggregatedTable("fra_modules_structure").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_structure_".add(fileprefix, "_"), true);
                }
                if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.template].Any())
                {
                    tRecord.frontierDLCDataTables[moduleIterationRecordSummary.template].GetAggregatedTable("fra_modules_template").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_template".add(fileprefix, "_"), true);
                }
                if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.diversity].Any())
                {
                    tRecord.frontierDLCDataTables[moduleIterationRecordSummary.diversity].GetAggregatedTable("fra_module_diversity").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_diversity_".add(fileprefix, "_"), true);
                }


                string finalOverviewPath = crawlerReportFolder.pathFor("fra_modules_impact".add(fileprefix, "_"), getWritableFileMode.newOrExisting);
                objectTable <moduleFinalOverview> finalOverview = new objectTable <moduleFinalOverview>(finalOverviewPath, false, "ModuleName", "module_impact");
                finalOverview.description = "Aggregate (DLC and iterations) metrics on modules' impact to the result.";

                aceDictionarySet <moduleIterationRecordSummary, moduleIterationRecord> moduleIterationsByModule = new aceDictionarySet <moduleIterationRecordSummary, moduleIterationRecord>();
                List <moduleIterationRecordSummary> moduleActive = new List <moduleIterationRecordSummary>();

                foreach (var wRecord in wRecords)
                {
                    dlc_c++;
                    foreach (var pair in wRecord.frontierDLC.modRecords)
                    {
                        moduleIterationsByModule.Add(pair.Value.moduleSummaryEnum, pair.Value.GetList());
                        if (!moduleActive.Contains(pair.Value.moduleSummaryEnum))
                        {
                            moduleActive.Add(pair.Value.moduleSummaryEnum);
                        }
                    }
                }

                int modC = 0;
                List <moduleFinalOverview> modList = new List <moduleFinalOverview>();
                foreach (var modType in moduleActive)
                {
                    moduleFinalOverview mfo = new moduleFinalOverview();
                    mfo.deploy(tRecord.instance.name, modType, moduleIterationsByModule[modType], dlc_c);
                    modC += moduleIterationsByModule[modType].Count;
                    finalOverview.AddOrUpdate(mfo);
                    modList.Add(mfo);
                }

                moduleFinalOverview mfoSum = new moduleFinalOverview();


                mfoSum.deploySum(tRecord.instance.name, modList);
                finalOverview.AddOrUpdate(mfoSum);

                foreach (var mfo in modList)
                {
                    mfo.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
                    finalOverview.AddOrUpdate(mfo);

                    record_moduleImpact.AddOrUpdate(mfo);
                }


                mfoSum.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
                record_moduleImpact.AddOrUpdate(mfoSum);

                //    finalOverview.SaveAs(finalOverviewPath.add(".xml"));
                DataTable fover = finalOverview.GetDataTable(null, mfoSum.Crawler);

                fover.SetAggregationOriginCount(modC);
                fover.SetAggregationAspect(dataPointAggregationAspect.onTableMultiRow);
                fover.GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_modules_impact_overview", true);


                record_moduleImpact.Save();
                var midt = record_moduleImpact.GetDataTable(null, "Module impacts");
                midt.AddExtra("The last benchmark metrics entry [" + imbWEMManager.index.experimentEntry.CrawlID + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());
                midt.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionReportFolder, imbWEMManager.authorNotation, "fra_modules_impact_".add(fileprefix, "_"));
            }
            else
            {
                dlc_c = tRecord.children.Count();
            }

            if (iterationTimelines.Any())
            {
                DataTable crawlTimeline = iterationTimelines.GetAggregatedTable("Crawler_Timeline", dataPointAggregationAspect.overlapMultiTable);
                crawlTimeline.SetDescription("Iteration-synced aggregated performance timeline using DLC records [" + wRecords.Count + "] domains.");
                crawlTimeline.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionCrawlerFolder, imbWEMManager.authorNotation, "timeline_performance_".add(imbWEMManager.index.experimentEntry.Crawler));
            }
            //String atl = "timeline_performance".add(tRecord.instance.name, "_").add("xml", ".");

            var domainPerfList = tRecord.lastDomainIterationTable.GetList();

            var benchmark = new reportPlugIn_benchmarkResults(); //records.GetOrCreate(imbWEMManager.index.experimentEntry.TestID);

            tRecord.performance.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);

            tRecord.performance.jobTimeInMinutes = tRecord.cpuTaker.GetTimeSpanInMinutes();

            record_performances.AddOrUpdate(tRecord.performance);

            benchmark.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);


            benchmark.CrawlTime = tRecord.cpuTaker.GetTimeSpanInMinutes(); //tRecord.cpuTaker.GetTimeSpan().TotalMinutes; //.timeFinish.Subtract(tRecord.timeStart).TotalMinutes;


            benchmark.IP           = domainPerfList.Average(x => x.IP);
            benchmark.IPnominal    = domainPerfList.Average(x => x.IPnominal);
            benchmark.IP_collected = domainPerfList.Average(x => x.IP_collected);
            benchmark.Lm_collected = domainPerfList.Average(x => x.Lm_collected);
            benchmark.Lm_recall    = domainPerfList.Average(x => x.Lm_recall);
            benchmark.E_PP         = domainPerfList.Average(x => x.E_PP);
            benchmark.E_TP         = domainPerfList.Average(x => x.E_TP);
            benchmark.IP_recall    = domainPerfList.Average(x => x.IP_recall);
            benchmark.Page_recall  = domainPerfList.Average(x => x.Page_recall);
            benchmark.Term_recall  = domainPerfList.Average(x => x.Term_recall);

            var resourcesamples = tRecord.measureTaker.GetLastSamples(1000);
            var lastsample      = tRecord.measureTaker.GetLastTake();

            benchmark.DataLoad = lastsample.bytesLoadedTotal / benchmark.CrawlTime;
            benchmark.CPU      = resourcesamples.Average(x => x.cpuRateOfProcess);
            benchmark.RAM      = resourcesamples.Average(x => x.physicalMemory);

            records.AddOrUpdate(benchmark);
            records.Save();

            var dt = records.GetDataTable(null, imbWEMManager.index.experimentEntry.CrawlID);

            dt.AddExtra("The last benchmark metrics entry [" + benchmark.Crawler + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());

            dt.SetAdditionalInfoEntry("DLC Threads - TC", __machine.maxThreads);
            dt.SetAdditionalInfoEntry("LoadTake - LT", tRecord.instance.settings.limitIterationNewLinks);
            dt.SetAdditionalInfoEntry("PageLoads - PL", tRecord.instance.settings.limitTotalPageLoad);
            dt.SetAdditionalInfoEntry("Sample size - DC", dlc_c);
            dt.SetAdditionalInfoEntry("Session ID", imbWEMManager.index.experimentEntry.SessionID);



            dt.GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "result", true);

            benchmark.GetUserManualSaved(crawlerReportFolder.pathFor("crawler\\result.txt"));

            //  crawlTimeline.saveObjectToXML(homeFolder.pathFor(atl));
            //  crawlTimeline.saveObjectToXML(reportFolder.pathFor(atl));

            // all three modules summary

            imbWEMManager.settings.directReportEngine.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_reportEngine.txt"));
            imbWEMManager.settings.crawlerJobEngine.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_crawlJobEngine.txt"));
            imbWEMManager.settings.executionLog.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_executionLogs.txt"));

            tRecord.instance.settings.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_crawler.txt"));
            record_performances.Save();
            var perfDT = record_performances.GetDataTable(null, imbWEMManager.index.experimentEntry.CrawlID);

            perfDT.AddExtra("The last benchmark metrics entry [" + benchmark.Crawler + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());

            perfDT.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionReportFolder, imbWEMManager.authorNotation, "crawl_performances", true);
        }
コード例 #14
0
        /// <summary>
        /// Registers the plugin.
        /// </summary>
        /// <param name="type">The type.</param>
        /// <param name="sourceDllPath">The source DLL path.</param>
        /// <param name="output">The output.</param>
        protected void registerPlugin(Type type, String sourceDllPath, ILogBuilder output)
        {
            if (!DOSHOWLOGS)
            {
                output = null;
            }
            if (!bannedShortNames.ContainsKey(type.Name))
            {
                if (pluginTypesByName.ContainsKey(type.Name))
                {
                    //if (output != null) output.log("Short-name registration of [" + type.Name + "] failed: name already occupied. You'll have to call both by [directory].[typename] path.");
                    bannedShortNames.Add(type.Name, type);

                    Type pair = null;
                    if (pluginTypesByName.TryRemove(type.Name, out pair))
                    {
                        bannedShortNames.Add(type.Name, pair);
                    }
                }
                else
                {
                    pluginTypesByName.Add(type.Name, type);
                    //if (output != null) output.log("Short-name registration of [" + type.Name + "] done.");
                }
            }
            else
            {
                //if (output != null) output.log("Short-name registration of [" + type.Name + "] failed, the name is banned from short-name registration for [" + bannedShortNames[type.Name] + "] plugins.");
            }

            if (!sourceDllPath.isNullOrEmpty())
            {
                String dirSufix = sourceDllPath;
                if (folderWithPlugins != null)
                {
                    dirSufix = sourceDllPath.removeStartsWith(folderWithPlugins.path).Replace(Path.DirectorySeparatorChar, '.');
                }
                dirSufix = dirSufix.Replace("/", ".");
                dirSufix = dirSufix.Replace("..", ".");

                String dirNamePath = dirSufix.add(type.Name, ".");

                if (pluginTypesByPathName.ContainsKey(dirNamePath))
                {
                    if (output != null)
                    {
                        output.log("[directory].[typename] (" + dirNamePath + ") registration of [" + type.Name + "] failed - can't have multiple plugins with the same name, in the same directory. Move it in sub folder or recompile under another class name");
                    }
                }
                else
                {
                    if (output != null)
                    {
                        output.log("[directory].[typename] (" + dirNamePath + ") registration of [" + type.Name + "] done. ");
                    }
                }
            }

            if (supportDirtyNaming)
            {
                String dirtyName = getDirtyForm(type.Name);
                if (!dirtyDictionary.ContainsKey(dirtyName))
                {
                    dirtyDictionary.Add(dirtyName, type);
                }
            }
        }