private void ProcessChildItems(List <ReportResultItem> items, string groupKey, int level, bool visibility)
{
if (items == null)
{
return;
}
for (int i = 0; i < items.Count; i++)
{
string newGroupKey = String.Format(CultureInfo.InvariantCulture, "{0}{1}_", groupKey, i);
ReportResultItem item = items[i];
bool hasChildren = (item.ChildItems != null && item.ChildItems.Count > 0);
TableRow row = new TableRow();
{
TableCell cell1 = new TableCell();
string imgUrl = ResolveClientUrl(hasChildren ? "~/layouts/images/plus.gif" : "~/layouts/images/dot.gif");
cell1.Text = String.Format(
CultureInfo.InvariantCulture,
"<span style='padding-left:{0}px;'><img src='{1}' border='0' width='9px' height='9px' /> {2}</span>",
level * 20,
imgUrl,
item.Name);
row.Cells.Add(cell1);
TableCell cell2 = new TableCell();
cell2.Text = item.Total.ToString("f");
cell2.HorizontalAlign = HorizontalAlign.Right;
cell2.Width = Unit.Pixel(100);
row.Cells.Add(cell2);
}
row.Style.Add(HtmlTextWriterStyle.Display, visibility ? "" : "none");
row.Attributes.Add("key", groupKey);
if (hasChildren)
{
row.Style.Add(HtmlTextWriterStyle.Cursor, "hand");
row.Attributes.Add("onclick",
String.Format(CultureInfo.InvariantCulture, "collapse_expand(this, '{0}');", newGroupKey));
}
MainTable.Rows.Add(row);
ProcessChildItems(item.ChildItems, newGroupKey, level + 1, false);
}
}
private void Level1Classification(List <DocumentData> TrainData, List <DocumentData> TestData)
{
var selectedFeatures = AppDataCenter.Singleton.FeaturesSelected.Select((a, i) => new { item = a, index = i })
.Where(a => a.item.IsSelected).Select(a => a.index).ToArray();
// selectedFeatures = new int[] { 37 };
ClassifierService.Service.InitSelectFeatures(selectedFeatures);
ScalingFactors scales = null;
// if(AppDataCenter.Singleton.PathToScale != null)
// {
// scales = FeatureListHolder.loadScalesFromFile(ClassifierService.Service.LocalFeatures, AppDataCenter.Singleton.PathToScale);
// }
// AppDataCenter.Singleton.AddConsoleMessage("Building Model");
AppDataCenter.Singleton.ChosenFeatures = ClassifierService.Service.LocalFeatures;
AppDataCenter.Singleton.UpdateFeatureVertHorzSelection();
AppDataCenter.Singleton.EntireModelStats = ClassifierService.Service.CreateModel(AppDataCenter.Singleton.ChosenFields
, TrainData, TestData, Configuration, m_featuresCalculate, AppDataCenter.Singleton.AddConsoleMessage, TuneScale, AppDataCenter.Singleton.PathToScale);
// if (AppDataCenter.Singleton.PathToScale != null)
// {
// FeatureListHolder.SaveScalesToFile(ClassifierService.Service.LocalFeatures, AppDataCenter.Singleton.PathToScale,scales);
// }
double[] confidenceOut;
List <ReportResultItem> ReoprtResults = new List <ReportResultItem>();
Array.ForEach(AppDataCenter.Singleton.ChosenFields,
new Action <string>(a =>
{
ReoprtResults.Add(new ReportResultItem(a));
}));
ReportResultItem NullResult = new ReportResultItem("");
ReoprtResults.Add(NullResult);
AppDataCenter.Singleton.RejectedFields.Clear();
ClassifierService.Service.level1RunTestForLevel2(TrainData, AppDataCenter.Singleton.ChosenFields, AppDataCenter.Singleton.AddConsoleMessage);
// add printing of train data sucss
ConcurrentBag <FieldReportItem> fieldBag = new ConcurrentBag <FieldReportItem>();
AppDataCenter.Singleton.AddConsoleMessage("Running Test on : " + TestData.Count + " Pages");
int index = 0;
// int result;
foreach (DocumentData doc1 in TestData)
{
var startTimeFeatures = DateTime.Now;
// AppDataCenter.Singleton.AddConsoleMessage("Testing Page : " + docnum++);
// List<FieldData> fields = ClassifierService.Service.GetFieldsOfDoc(FieldsToTest, doc1);
TimeSpan FeatureCalcSpan = new TimeSpan(0);
/* foreach (CandidateData field in doc1.Candidates)
* {
* result = 0;
* startTimeFeatures = DateTime.Now;
* if (m_useLastRuntimeData == true)
* {
* foreach (var featureIndex in ClassifierService.Service.LocalFeatures
* .Select((x, i) => AppDataCenter.Singleton.IsFeatureCalculate(x.Name) == true ? i : -1)
* .Where(x => x >= 0))
* {
* field.Features[featureIndex] = ClassifierService.Service.GetFieldFeature(doc1, field, featureIndex);
* }
* }
* else
* {
* field.Features = ClassifierService.Service.GetFieldFeatures(doc1, field);
* }
* FeatureCalcSpan += (DateTime.Now - startTimeFeatures);
* result = ClassifierService.Service.GetDescition(field.Features, out confidenceOut);
*
*
* // field.AccordConfidance = (ClassifierService.Service.NormalizaedConfidence(confidenceOut));
* field.AccordConfidance = (confidenceOut);
* }*/
List <int> featursIndexFilter = new List <int>();
// Get the index of the feature we need to calculate - all other features will be use the value already exist in the field
ClassifierService.Service.getDocFeatures(doc1, null, null, AppDataCenter.Singleton.ChosenFields, featursIndexFilter, false, null, true, true);
double NumGolden = 0;
double NumMatched = 0;
foreach (var r in AppDataCenter.Singleton.ChosenFields.Select((x, i) => new { Value = x, Index = i }))
{
// if (doc1.Candidates.Where(a => a.NameFromTypist == r.Value).ToList().Count() <= 0) continue;
NumGolden++;
CandidateData fieldCandidate = doc1.Candidates.OrderByDescending(a => a.AccordConfidance[r.Index]).FirstOrDefault();
FieldReportItem fieldReportItem = new FieldReportItem(r.Value, r.Index, fieldCandidate, doc1);
if (fieldReportItem.IsMatch)
{
NumMatched++;
}
/* }
* else
* {
* fieldReportItem.isRejected = true;
*
* }*/
if (!fieldReportItem.IsMatch)
{
Trace.WriteLine("Doc : " + doc1.DocumentName + " ,Confidance : " + fieldCandidate.AccordConfidance[r.Index] + ",Field : " + r.Value + " ,Recognized : " + fieldCandidate.NameFromTypist + " content : " + fieldCandidate.Content + (fieldReportItem.IsMatch ? " True" : " False"));
}
fieldBag.Add(fieldReportItem);
}
AppDataCenter.Singleton.AddConsoleMessage(index++ + " Page : " + doc1.DocumentName + " Getting Features (sec) : " + FeatureCalcSpan.TotalSeconds + " success : " + Math.Round(NumMatched / NumGolden * 100) + "%");
}
AppDataCenter.Singleton.AddConsoleMessage("Update results");
AppDataCenter.Singleton.EntireModelStats.testresults = fieldBag.ToList();
ReportData.noOfFields = fieldBag.Count();
ReportData.NoOfPages = TestData.Count;
ReportData.fpCount = fieldBag.Where(a => a.IsFP).Count();
ReportData.matchCount = fieldBag.Where(a => a.IsMatch).Count();
ReportData.rejectCount = fieldBag.Where(a => a.IsRejected).Count();
foreach (FieldReportItem fieldresult in fieldBag.Where(a => a.IsMatch))
{
ReportData.QualityMatch.Add(fieldresult.MatchQuality);
}
List <FieldReportItem> FPList = fieldBag.Where(a => a.IsFP).ToList().OrderBy(b => b.Field.NameFromTypist ?? "").ThenByDescending(a => a.Field.AccordConfidance.Max()).ToList();
foreach (var r in AppDataCenter.Singleton.ChosenFields)
{
List <FieldReportItem> FPfromGolden = FPList.Where(a => a.ExpectedField.NameFromTypist == r).ToList();
List <FieldReportItem> FPnotFromGolden = FPList.Where(a => a.ExpectedField.NameFromTypist != r && a.MatchedName == r).ToList();
double success = (double)fieldBag.Where(a => a.Field.NameFromTypist == r && a.IsMatch).Count() / (double)fieldBag.Where(a => a.Field.NameFromTypist == r).Count() * 100;
int countNum = fieldBag.Where(a => a.Field.NameFromTypist == r).Count();
Trace.WriteLine("*********** The field : " + r + " success : " + Math.Round(success, 1) + "% Count : " + countNum + " *************");
}
AppDataCenter.Singleton.NotifyChange(NotifyGroup.ScatterData);
}
