public async Task <ActionResult> Update(ClassificationModel model)
{
if (!_permissionService.Authorize(PermissionProvider.ManageDepartment))
{
return(AccessDeniedView());
}
var classification = await _classificationService.GetByIdAsync(model.Id);
if (classification == null)
{
throw new ArgumentException("No classification found with the specified id");
}
var existedClassification = await _classificationService.GetClassificationByClassificationCode(model.Code);
if (existedClassification != null && existedClassification.Id != classification.Id)
{
return(Content("Classification Code has Existed!"));
}
classification.Code = model.Code;
classification.Name = model.Name;
classification.Description = model.Description;
classification.Severity = model.Severity;
classification.Dectability = model.Dectability;
classification.FoundByFunctionId = model.FoundByFunction.Id == 0 ? null : (int?)model.FoundByFunction.Id;
await _classificationService.UpdateAsync(classification);
return(Json(new
{
status = "success",
}));
}
public ClassificationModel GetClassificationModelWithClassificationTVItemIDDB(int ClassificationTVItemID)
{
ClassificationModel ClassificationModel = (from c in db.Classifications
let classTVText = (from cl in db.TVItemLanguages where cl.Language == (int)LanguageRequest && cl.TVItemID == c.ClassificationTVItemID select cl.TVText).FirstOrDefault <string>()
where c.ClassificationTVItemID == ClassificationTVItemID
select new ClassificationModel
{
Error = "",
ClassificationID = c.ClassificationID,
DBCommand = (DBCommandEnum)c.DBCommand,
ClassificationTVItemID = c.ClassificationTVItemID,
ClassificationTVText = classTVText,
ClassificationType = (ClassificationTypeEnum)c.ClassificationType,
Ordinal = c.Ordinal,
LastUpdateDate_UTC = c.LastUpdateDate_UTC,
LastUpdateContactTVItemID = c.LastUpdateContactTVItemID,
}).FirstOrDefault <ClassificationModel>();
if (ClassificationModel == null)
{
return(ReturnError(string.Format(ServiceRes.CouldNotFind_With_Equal_, ServiceRes.Classification, ServiceRes.ClassificationTVItemID, ClassificationTVItemID)));
}
return(ClassificationModel);
}
public override Task TrainAsync(ClassificationModel classificationModel)
{
int numFeatures = classificationModel.FeatureVectors.Count;
double[][] input = new double[numFeatures][];
int[] responses = new int[numFeatures];
for (int featureIndex = 0;
featureIndex < classificationModel.FeatureVectors.Count;
++featureIndex)
{
var featureVector = classificationModel.FeatureVectors[featureIndex];
input[featureIndex] = Array.ConvertAll(featureVector.FeatureVector.BandIntensities, s => (double)s / ushort.MaxValue);
responses[featureIndex] = (int)featureVector.FeatureClass;
}
NaiveBayesLearning <NormalDistribution> learning = new NaiveBayesLearning <NormalDistribution>();
return(Task.Factory.StartNew(() =>
{
learning.Options.InnerOption = new NormalOptions()
{
Regularization = 1e-5
};
_bayes = learning.Learn(input, responses);
}));
}
public async Task <ActionResult> Create(ClassificationModel classificationModel)
{
if (!_permissionService.Authorize(PermissionProvider.ManageDepartment))
{
return(AccessDeniedView());
}
var existedClassification = await _classificationService.GetClassificationByClassificationCode(classificationModel.Code);
if (existedClassification != null)
{
return(Content("Classification Code has Existed!"));
}
var classification = new Classification()
{
Code = classificationModel.Code,
Name = classificationModel.Name,
Description = classificationModel.Description,
Severity = classificationModel.Severity,
Dectability = classificationModel.Dectability,
FoundByFunctionId = classificationModel.FoundByFunction.Id == 0 ? null : (int?)classificationModel.FoundByFunction.Id
};
await _classificationService.InsertAsync(classification);
//return Json(classification);
return(Json(new
{
status = "success",
}));
}
public ClassificationModel PostDeleteClassificationWithClassificationTVItemIDDB(int ClassificationTVItemID)
{
ContactOK contactOK = IsContactOK();
if (!string.IsNullOrEmpty(contactOK.Error))
{
return(ReturnError(contactOK.Error));
}
Classification ClassificationToDelete = GetClassificationWithClassificationWithClassificationTVItemIDDB(ClassificationTVItemID);
if (ClassificationToDelete == null)
{
return(ReturnError(string.Format(ServiceRes.CouldNotFind_ToDelete, ServiceRes.Classification)));
}
ClassificationModel ClassificationModelRet = PostDeleteClassificationDB(ClassificationToDelete.ClassificationID);
if (!string.IsNullOrWhiteSpace(ClassificationModelRet.Error))
{
return(ReturnError(ClassificationModelRet.Error));
}
return(ReturnError(""));
}
public async Task <IndexResult> Index(string id, ClassificationModel existing)
{
var apiKey = Environment.GetEnvironmentVariable("RIJKSMUSEUM_DATA_API_KEY");
var collectionApiRequestUrl = $"https://www.rijksmuseum.nl/api/en/collection/{id}?key={apiKey}";
var collectionResponse = await HttpClient.GetStringAsync(collectionApiRequestUrl);
var collectionJson = JObject.Parse(collectionResponse);
var model = new ClassificationModel
{
Source = Source,
PageId = id,
SourceLink = $"https://www.rijksmuseum.nl/en/collection/{id}",
OriginalArtist = collectionJson["artObject"]["principalMaker"].Value <string>(),
Date = collectionJson["artObject"]["dating"]["presentingDate"].Value <string>(),
Name = collectionJson["artObject"]["titles"].Values <string>().FirstOrDefault()
};
if (string.IsNullOrWhiteSpace(model.Name))
{
model.Name = collectionJson["artObject"]["title"].Value <string>();
}
var indexResult = new IndexResult
{
Model = model,
ImageJpeg = await new TileImageStitcher().GetStitchedTileImageJpegBytes(id, Environment.GetEnvironmentVariable("RIJKSMUSEUM_DATA_API_KEY"))
};
return(indexResult);
}
// Check
public string ClassificationModelOK(ClassificationModel classificationModel)
{
string retStr = FieldCheckNotZeroInt(classificationModel.ClassificationTVItemID, ServiceRes.ClassificationTVItemID);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(retStr);
}
retStr = FieldCheckNotNullAndMinMaxLengthString(classificationModel.ClassificationTVText, ServiceRes.ClassificationTVText, 3, 200);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(retStr);
}
retStr = _BaseEnumService.ClassificationTypeOK(classificationModel.ClassificationType);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(retStr);
}
retStr = FieldCheckNotNullAndWithinRangeInt(classificationModel.Ordinal, ServiceRes.Ordinal, 0, 10000);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(retStr);
}
retStr = _BaseEnumService.DBCommandOK(classificationModel.DBCommand);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(retStr);
}
return("");
}
public async Task <IndexResult> Index(string id, ClassificationModel existing)
{
var sourceLink = $"https://www.pop.culture.gouv.fr/notice/joconde/{id}";
var htmlDoc = await new IndexingHttpClient().GetPage(HttpClient, sourceLink, Logging);
if (htmlDoc == null)
{
return(null);
}
var model = DetailsParser.ParseHtmlToNewModel(Source, id, sourceLink, htmlDoc);
var imageLinkNodes = htmlDoc.DocumentNode
.SelectNodes("//div[@class='jsx-241519627 fieldImages']//img");
if (imageLinkNodes == null)
{
return(null);
}
var imageLink = HttpUtility.HtmlDecode(imageLinkNodes.First().Attributes["src"].Value);
var imageBytes = await new IndexingHttpClient().GetImage(HttpClient, imageLink, Logging);
if (imageBytes == null)
{
return(null);
}
return(new IndexResult
{
Model = model,
ImageJpeg = Image.Load <Rgba64>(imageBytes)
});
}
public void AddNewVote(ClassificationModel Classification)
{
if (Classification == null)
{
throw new NullReferenceException();
}
if (Classification.Point != null)
{
int value = GetClassificationForPoint(Classification.Point).Where(x => x.User.Id == Classification.User.Id).Count();
if (value > 0)
{
return;
}
}
using (AppTourEntities data = new AppTourEntities())
{
CLASSIFICATION _new = new CLASSIFICATION
{
ID = (Classification.Id == null || Classification.Id == Guid.Empty ? Guid.NewGuid() : Classification.Id),
EVENT = (Classification.Event != null ? data.EVENT.SingleOrDefault(x => x.ID.Equals(Classification.Event.Id)) : null),
POINT = (Classification.Point != null ? data.POINT.SingleOrDefault(x => x.ID.Equals(Classification.Point.Id)) : null),
CLASSIFICATION1 = Classification.Classification,
USER = data.USER.SingleOrDefault(x => x.ID.Equals(Classification.User.Id)),
CREATION_DATE = DateTime.Now
};
data.CLASSIFICATION.AddObject(_new);
data.SaveChanges();
}
}
public ActionResult InsertVote(string userId, string pointId, string Classifications)
{
if (userId == null || Classifications == null || pointId == null || userId == string.Empty ||
Classifications == string.Empty || pointId == string.Empty)
{
return(Content("error"));
}
if (isGuid.IsMatch(userId) && isGuid.IsMatch(pointId))
{
ClassificationModel classification = new ClassificationModel
{
Id = Guid.Empty,
Point = new PointService().GetPoint(new Guid(pointId)),
Classification = Convert.ToInt32(Classifications),
User = new UserService().GetUser(new Guid(userId))
};
if (classification.IsValid)
{
new ClassificationService().AddVote(classification);
return(Content("sucesso"));
}
return(Content("Model Not Valid: " + classification.Error));
}
return(RedirectToAction("Empty"));
}
public async Task <string> SendToElasticSearch(ClassificationModel classification)
{
var json = JObject.Parse(JsonConvert.SerializeObject(classification, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
}));
var path = "/classification/_doc/" + HttpUtility.UrlEncode($"{classification.Source}:{classification.PageId}");
return(await SendToElasticSearch(HttpMethod.Post, path, json));
}
public ClassificationModel TrainGroupWise(BaseVector[] x, int[][] y, int ngroups, IGroupDataProvider data,
int nthreads)
{
ClassificationModel[] c = new ClassificationModel[ngroups];
ThreadDistributor td = new ThreadDistributor(nthreads, ngroups, i => { c[i] = TrainGroupWise(i, y, x, data); });
td.Start();
return(new GroupWiseClassifier(c));
}
public bool GetIsItSameObject(ClassificationModel ClassificationModel, TVItemModel tvItemModelClassificationExit)
{
bool IsSame = false;
if (ClassificationModel.ClassificationTVItemID == tvItemModelClassificationExit.TVItemID)
{
IsSame = true;
}
return(IsSame);
}
public ActionResult List()
{
var model = new ClassificationModel();
int recordCount = 0;
var cflist = ClassificationBLL.Instance.Classification_GetList(1, 0, "sort asc", "parentid=0", out recordCount);
var sectionlist = SectionBLL.Instance.Section_GetList(0);
model.CFList = cflist.ToList();
model.SectionList = sectionlist.ToList();
return(View(model));
}
/// <summary>
///
/// </summary>
private async void ComputeConfusionMatrixAsync()
{
var landCoverTypes = _mainWindowViewModel.LandcoverTypes.Values.Select(t => t.Name).ToList();
var model = new ClassificationModel(ProjectionName, landCoverTypes, _mainWindowViewModel.Layers.ToList(), FeaturesViewModel.AllFeaturesView, _mainWindowViewModel.LandcoverTypes.Values.ToImmutableList());
var confusionMatrices = await CurrentClassifierViewModel.ComputeFoldedConfusionMatrixAsync(model, 10);
PredictionAccuracyDialog dialog = new PredictionAccuracyDialog();
dialog.ShowDialog(confusionMatrices);
}
public void SetMetaData(ClassificationModel model)
{
Console.WriteLine("Getting metadata for page id " + model.PageId);
var html = NgaDataAccess.GetAssetDetails(model.PageId);
var details = AssetDetailsParser.ParseHtmlToNewModel(html);
model.OriginalArtist = details.OriginalArtist;
model.Artist = details.Artist;
model.Name = details.Name;
model.Date = details.Date;
model.SourceLink = details.SourceLink;
}
// Post
//public ClassificationModel ClassificationAddOrModifyDB(FormCollection fc)
//{
// int tempInt = 0;
// int ParentTVItemID = 0;
// int ClassificationTVItemID = 0;
// bool IsActive = false;
// bool IsPointSource = false;
// double Lat = 0.0D;
// double Lng = 0.0D;
// ContactOK contactOK = IsContactOK();
// if (!string.IsNullOrWhiteSpace(contactOK.Error))
// return ReturnError(contactOK.Error);
// if (string.IsNullOrWhiteSpace(fc["ParentTVItemID"]))
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.ParentTVItemID));
// int.TryParse(fc["ParentTVItemID"], out ParentTVItemID);
// if (ParentTVItemID == 0)
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.ParentTVItemID));
// if (string.IsNullOrWhiteSpace(fc["ClassificationTVItemID"]))
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.ClassificationTVItemID));
// int.TryParse(fc["ClassificationTVItemID"], out ClassificationTVItemID);
// // ClassificationTVItemID == 0 ==> Add
// // ClassificationTVItemID > 0 ==> Modify
// TVItemModel tvItemModelPolSource = null;
// if (ClassificationTVItemID != 0)
// {
// tvItemModelPolSource = _TVItemService.GetTVItemModelWithTVItemIDDB(ClassificationTVItemID);
// if (!string.IsNullOrWhiteSpace(tvItemModelPolSource.Error))
// return ReturnError(tvItemModelPolSource.Error);
// }
// ClassificationModel ClassificationNewOrToChange = new ClassificationModel();
// if (ClassificationTVItemID != 0)
// {
// ClassificationNewOrToChange = GetClassificationModelWithClassificationTVItemIDDB(ClassificationTVItemID);
// if (!string.IsNullOrWhiteSpace(ClassificationNewOrToChange.Error))
// return ReturnError(ClassificationNewOrToChange.Error);
// }
// if (string.IsNullOrWhiteSpace(fc["IsActive"]))
// IsActive = false;
// else
// IsActive = true;
// if (!IsActive)
// {
// int.TryParse(fc["InactiveReason"], out tempInt);
// if (tempInt == 0)
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.InactiveReason));
// ClassificationNewOrToChange.InactiveReason = (PolSourceInactiveReasonEnum)tempInt;
// }
// else
// {
// ClassificationNewOrToChange.InactiveReason = null;
// }
// if (string.IsNullOrWhiteSpace(fc["IsPointSource"]))
// IsPointSource = false;
// else
// IsPointSource = true;
// ClassificationNewOrToChange.IsPointSource = IsPointSource;
// double.TryParse(fc["Lat"], out Lat);
// if (Lat == 0.0D)
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.Lat));
// double.TryParse(fc["Lng"], out Lng);
// if (Lng == 0.0D)
// return ReturnError(string.Format(ServiceRes._IsRequired, ServiceRes.Lng));
// List<Coord> coordList = new List<Coord>() { new Coord() { Lat = (float)Lat, Lng = (float)Lng, Ordinal = 0 } };
// using (TransactionScope ts = new TransactionScope())
// {
// string ObservationInfo = ((int)PolSourceObsInfoEnum.SourceStart).ToString() + ",";
// List<int> obsIntList = ObservationInfo.Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
// string ObservationLanguageTVText = ServiceRes.Error;
// string TVText = _BaseEnumService.GetEnumText_PolSourceObsInfoTextEnum(PolSourceObsInfoEnum.Error);
// int NextSiteNumber = 0;
// if (ClassificationTVItemID == 0)
// {
// NextSiteNumber = GetNextAvailableSiteNumberWithParentTVItemIDDB(ParentTVItemID);
// ClassificationNewOrToChange.Site = NextSiteNumber;
// TVText = TVText + " - " + "000000".Substring(0, "000000".Length - NextSiteNumber.ToString().Length) + NextSiteNumber.ToString();
// TVItemModel tvItemModelNewClassification = _TVItemService.PostAddChildTVItemDB(ParentTVItemID, TVText, TVTypeEnum.Classification);
// if (!string.IsNullOrWhiteSpace(tvItemModelNewClassification.Error))
// return ReturnError(tvItemModelNewClassification.Error);
// ClassificationNewOrToChange.ClassificationTVItemID = tvItemModelNewClassification.TVItemID;
// ClassificationNewOrToChange.ClassificationTVText = TVText;
// ClassificationNewOrToChange = PostAddClassificationDB(ClassificationNewOrToChange);
// if (!string.IsNullOrWhiteSpace(ClassificationNewOrToChange.Error))
// return ReturnError(ClassificationNewOrToChange.Error);
// // Automatically add one Pollution Source Observation for today
// PolSourceObservationModel polSourceObservationModelNew = new PolSourceObservationModel()
// {
// ClassificationID = ClassificationNewOrToChange.ClassificationID,
// ObservationDate_Local = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day),
// ContactTVItemID = contactOK.ContactTVItemID,
// Observation_ToBeDeleted = "",
// };
// PolSourceObservationModel polSourceObservationModelRet = _PolSourceObservationService.PostAddPolSourceObservationDB(polSourceObservationModelNew);
// if (!string.IsNullOrWhiteSpace(polSourceObservationModelRet.Error))
// return ReturnError(polSourceObservationModelRet.Error);
// // Automatically add one Pollution Source Observation Issue
// PolSourceObservationIssueModel polSourceObservationIssueModelNew = new PolSourceObservationIssueModel();
// polSourceObservationIssueModelNew.PolSourceObservationID = polSourceObservationModelRet.PolSourceObservationID;
// polSourceObservationIssueModelNew.ObservationInfo = ObservationInfo;
// polSourceObservationIssueModelNew.Ordinal = 0;
// PolSourceObservationIssueModel polSourceObservationIssueModelRet = _PolSourceObservationService._PolSourceObservationIssueService.PostAddPolSourceObservationIssueDB(polSourceObservationIssueModelNew);
// if (!string.IsNullOrWhiteSpace(polSourceObservationIssueModelRet.Error))
// return ReturnError(polSourceObservationIssueModelRet.Error);
// // doing the other language
// foreach (LanguageEnum lang in LanguageListAllowable.Where(c => c != LanguageRequest))
// {
// TVItemService tvItemService = new TVItemService(lang, _TVItemService.User);
// Thread.CurrentThread.CurrentCulture = new CultureInfo(lang + "-CA");
// Thread.CurrentThread.CurrentUICulture = new CultureInfo(lang + "-CA");
// ObservationInfo = ((int)PolSourceObsInfoEnum.SourceStart).ToString() + ",";
// ObservationLanguageTVText = ServiceRes.Error;
// TVText = _BaseEnumService.GetEnumText_PolSourceObsInfoTextEnum(PolSourceObsInfoEnum.Error);
// TVText = (string.IsNullOrWhiteSpace(TVText) ? ServiceRes.Error : TVText);
// if (ClassificationTVItemID == 0)
// {
// TVText = TVText + " - " + "000000".Substring(0, "000000".Length - NextSiteNumber.ToString().Length) + NextSiteNumber.ToString();
// }
// else
// {
// TVText = TVText + " - " + "000000".Substring(0, "000000".Length - ClassificationNewOrToChange.Site.ToString().Length) + ClassificationNewOrToChange.Site.ToString();
// }
// TVItemLanguageModel tvItemLanguageModel = new TVItemLanguageModel();
// tvItemLanguageModel.Language = lang;
// tvItemLanguageModel.TVText = TVText;
// tvItemLanguageModel.TVItemID = ClassificationNewOrToChange.ClassificationTVItemID;
// TVItemLanguageModel tvItemLanguageModelRet = tvItemService._TVItemLanguageService.PostUpdateTVItemLanguageDB(tvItemLanguageModel);
// if (!string.IsNullOrWhiteSpace(tvItemLanguageModelRet.Error))
// return ReturnError(tvItemLanguageModelRet.Error);
// Thread.CurrentThread.CurrentCulture = new CultureInfo(LanguageRequest + "-CA");
// Thread.CurrentThread.CurrentUICulture = new CultureInfo(LanguageRequest + "-CA");
// }
// }
// else
// {
// ClassificationNewOrToChange = PostUpdateClassificationDB(ClassificationNewOrToChange);
// if (!string.IsNullOrWhiteSpace(ClassificationNewOrToChange.Error))
// return ReturnError(ClassificationNewOrToChange.Error);
// }
// // Adding map info
// List<MapInfoPointModel> mapInfoPointModelList = _MapInfoService._MapInfoPointService.GetMapInfoPointModelListWithTVItemIDAndTVTypeAndMapInfoDrawTypeDB(ClassificationNewOrToChange.ClassificationTVItemID, TVTypeEnum.Classification, MapInfoDrawTypeEnum.Point);
// if (mapInfoPointModelList.Count == 0)
// {
// MapInfoModel mapInfoModelRet = _MapInfoService.CreateMapInfoObjectDB(coordList, MapInfoDrawTypeEnum.Point, TVTypeEnum.Classification, ClassificationNewOrToChange.ClassificationTVItemID);
// if (!string.IsNullOrWhiteSpace(mapInfoModelRet.Error))
// return ReturnError(mapInfoModelRet.Error);
// }
// else
// {
// mapInfoPointModelList[0].Lat = coordList[0].Lat;
// mapInfoPointModelList[0].Lng = coordList[0].Lng;
// MapInfoPointModel mapInfoPointModelRet = _MapInfoService._MapInfoPointService.PostUpdateMapInfoPointDB(mapInfoPointModelList[0]);
// if (!string.IsNullOrWhiteSpace(mapInfoPointModelRet.Error))
// return ReturnError(mapInfoPointModelRet.Error);
// }
// TVItemModel tvItemModelClassification = _TVItemService.GetTVItemModelWithTVItemIDDB(ClassificationNewOrToChange.ClassificationTVItemID);
// if (!string.IsNullOrWhiteSpace(tvItemModelClassification.Error))
// return ReturnError(tvItemModelClassification.Error);
// tvItemModelClassification.IsActive = IsActive;
// TVItemModel tvItemModelRet = _TVItemService.PostUpdateTVItemDB(tvItemModelClassification);
// if (!string.IsNullOrWhiteSpace(tvItemModelRet.Error))
// return ReturnError(tvItemModelRet.Error);
// ts.Complete();
// }
// return ClassificationNewOrToChange;
//}
public ClassificationModel PostAddClassificationDB(ClassificationModel ClassificationModel)
{
string retStr = ClassificationModelOK(ClassificationModel);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(ReturnError(retStr));
}
ContactOK contactOK = IsContactOK();
if (!string.IsNullOrEmpty(contactOK.Error))
{
return(ReturnError(contactOK.Error));
}
TVItemModel tvItemModelExist = _TVItemService.GetTVItemModelWithTVItemIDDB(ClassificationModel.ClassificationTVItemID);
if (!string.IsNullOrWhiteSpace(tvItemModelExist.Error))
{
return(ReturnError(tvItemModelExist.Error));
}
Classification ClassificationNew = new Classification();
retStr = FillClassification(ClassificationNew, ClassificationModel, contactOK);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(new ClassificationModel()
{
Error = retStr
});
}
using (TransactionScope ts = new TransactionScope())
{
db.Classifications.Add(ClassificationNew);
retStr = DoAddChanges();
if (!string.IsNullOrWhiteSpace(retStr))
{
return(ReturnError(retStr));
}
LogModel logModel = _LogService.PostAddLogForObj("Classifications", ClassificationNew.ClassificationID, LogCommandEnum.Add, ClassificationNew);
if (!string.IsNullOrWhiteSpace(logModel.Error))
{
return(ReturnError(logModel.Error));
}
ts.Complete();
}
return(GetClassificationModelWithClassificationIDDB(ClassificationNew.ClassificationID));
}
public override Task TrainAsync(ClassificationModel classificationModel)
{
int numFeatures = classificationModel.FeatureVectors.Count;
DecisionVariable[] decisionVariables = Enumerable.ToArray(classificationModel.Bands.Select(b => DecisionVariable.Continuous(b.ToString())));
double[][] input = new double[numFeatures][];
int[] responses = new int[numFeatures];
for (int featureIndex = 0; featureIndex < classificationModel.FeatureVectors.Count; ++featureIndex)
{
var featureVector = classificationModel.FeatureVectors[featureIndex];
input[featureIndex] = Array.ConvertAll(featureVector.FeatureVector.BandIntensities, s => (double)s / ushort.MaxValue);
responses[featureIndex] = featureVector.FeatureClass;
}
if (Boosting)
{
return(Task.Factory.StartNew(() =>
{
var classifier = new Boost <Weak <DecisionTree> >();
var teacher = new AdaBoostM1 <Weak <DecisionTree> >(classifier)
{
Creation = (weights) =>
{
var tree = new DecisionTree(decisionVariables, Enum.GetValues(typeof(LandcoverTypeViewModel)).Length);
var c45Learning = new C45Learning(tree);
c45Learning.Learn(input, responses, weights);
return new Weak <DecisionTree>(tree, (s, x) => s.Decide(x));
},
Iterations = Iterations,
Tolerance = 1e-2
};
teacher.Run(input, responses);
_tree = Either.Right <DecisionTree, Boost <Weak <DecisionTree> > >(classifier);
}));
}
else
{
return(Task.Factory.StartNew(() =>
{
var tree = new DecisionTree(decisionVariables, Enum.GetValues(typeof(LandcoverTypeViewModel)).Length);
C45Learning id3Learning = new C45Learning(tree);
id3Learning.Learn(input, responses);
_tree = Either.Left <DecisionTree, Boost <Weak <DecisionTree> > >(tree);
}));
}
}
// View rule at selected row
private void buttonXGetRule_Click(object sender, EventArgs e)
{
if (!canGetRule())
{
return;
}
int rowIndex = this.dataGridViewXDiagnosis.CurrentCell.RowIndex;
ClassificationModel treeModel = this.modelList[LearningAlgorithm.C45];
textBoxXDiagnosis.Text = Compute(this.trainningDataTabDianosis.TrainningAttributes[rowIndex]
, (treeModel as C45Model).Tree);
}
public async Task <IndexResult> Index(string id, ClassificationModel existing)
{
var sourceLink = $"https://www.moma.org/collection/works/{id}";
var htmlDoc = await new IndexingHttpClient().GetPage(HttpClient, sourceLink, Logging);
if (htmlDoc == null)
{
return(null);
}
var model = new ClassificationModel {
Source = Source, SourceLink = sourceLink, PageId = id
};
var infoNodes = htmlDoc.DocumentNode
.SelectNodes("//div[@class='work__short-caption']/h1/span");
if (infoNodes != null && infoNodes.Count > 0)
{
model.OriginalArtist = infoNodes[0].InnerText.Trim();
}
if (infoNodes != null && infoNodes.Count > 1)
{
model.Name = infoNodes[1].InnerText.Trim();
}
if (infoNodes != null && infoNodes.Count > 2)
{
model.Date = infoNodes[2].InnerText.Trim();
}
var imageLinkNodes = htmlDoc.DocumentNode
.SelectNodes("//img[@class='link/enable link/focus picture/image']");
if (imageLinkNodes == null)
{
return(null);
}
var imageLink = HttpUtility.HtmlDecode(imageLinkNodes.First().Attributes["data-image-overlay-src"].Value);
var imageBytes = await new IndexingHttpClient().GetImage(HttpClient, $"https://www.moma.org{imageLink}", Logging);
if (imageBytes == null)
{
return(null);
}
return(new IndexResult
{
Model = model,
ImageJpeg = Image.Load <Rgba64>(imageBytes)
});
}
// buttonXCreateModel click event
private void buttonXCreateModel_Click(object sender, EventArgs e)
{
DialogResult dialogResult;
DataTable orginalTrainningTable = this.dataGridViewXTrainning.DataSource as DataTable;
DataTable tableForTrainning;
int numberOfTrainningRows = 0;
// Get number of row form percent
if (this.comboBoxExTrainningDataPercent.SelectedValue != null)
{
numberOfTrainningRows = Convert.ToInt32(Math.Truncate(
((int)comboBoxExTrainningDataPercent.SelectedValue * 1.0 * orginalTrainningTable.Rows.Count) / 100));
}
// Set trainning data
if (numberOfTrainningRows == 0)
{
tableForTrainning = orginalTrainningTable;
}
else
{
var query = orginalTrainningTable.AsEnumerable().Take(numberOfTrainningRows);
tableForTrainning = query.CopyToDataTable <DataRow>();
}
TrainningData trainningData = new TrainningData(tableForTrainning, this.codification);
Properties.Settings.Default.negativeValue = trainningData.NegativeValue;
// Ask user what to do when selected model already exists
if (this.HaveModel())
{
dialogResult = MessageBox.Show("Mô hình này đã có. Bạn có muốn tạo mô hình mới?", "Lỗi",
MessageBoxButtons.YesNo, MessageBoxIcon.Warning);
if (dialogResult == DialogResult.No || dialogResult == DialogResult.Cancel)
{
return;
}
}
// Check dictionary already have active model. If not add new model
if (!this.modelList.ContainsKey(this.activeLearningAlgorithm))
{
this.modelList.Add(this.activeLearningAlgorithm, ClassificationModel.CreateModel(this.activeLearningAlgorithm));
}
// Trainning model
this.modelList[activeLearningAlgorithm].TrainningModel(trainningData);
}
public ClassificationModel PostUpdateClassificationDB(ClassificationModel ClassificationModel)
{
string retStr = ClassificationModelOK(ClassificationModel);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(ReturnError(retStr));
}
ContactOK contactOK = IsContactOK();
if (!string.IsNullOrEmpty(contactOK.Error))
{
return(ReturnError(contactOK.Error));
}
Classification ClassificationToUpdate = GetClassificationWithClassificationIDDB(ClassificationModel.ClassificationID);
if (ClassificationToUpdate == null)
{
return(ReturnError(string.Format(ServiceRes.CouldNotFind_ToUpdate, ServiceRes.Classification)));
}
retStr = FillClassification(ClassificationToUpdate, ClassificationModel, contactOK);
if (!string.IsNullOrWhiteSpace(retStr))
{
return(ReturnError(retStr));
}
using (TransactionScope ts = new TransactionScope())
{
retStr = DoUpdateChanges();
if (!string.IsNullOrWhiteSpace(retStr))
{
return(ReturnError(retStr));
}
LogModel logModel = _LogService.PostAddLogForObj("Classifications", ClassificationToUpdate.ClassificationID, LogCommandEnum.Change, ClassificationToUpdate);
if (!string.IsNullOrWhiteSpace(logModel.Error))
{
return(ReturnError(logModel.Error));
}
ts.Complete();
}
return(GetClassificationModelWithClassificationIDDB(ClassificationToUpdate.ClassificationID));
}
public void MoveForReview(
IAmazonDynamoDB dbClient,
ElasticSearchClient elasticSearchClient,
IAmazonS3 s3Client,
ClassificationModel model)
{
var json = JObject.FromObject(model, new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore
});
var result = dbClient.PutItemAsync(
NationalGalleryOfArtIndexer.TABLE_REVIEW,
Document.FromJson(json.ToString()).ToAttributeMap()
).Result;
if (result.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception("Failed to move to review table");
}
MoveS3ImageForReview(s3Client, model);
try
{
var searchDeletionResult = elasticSearchClient.DeleteFromElasticSearch(model).Result;
var searchDeletionResultJson = JObject.Parse(searchDeletionResult);
if (!string.Equals(searchDeletionResultJson["result"].Value <string>(), "deleted", StringComparison.OrdinalIgnoreCase))
{
throw new Exception("Failed to delete from elastic search");
}
}
catch (AggregateException exception)
{
if (!exception.Message.Contains("404 (Not Found)", StringComparison.OrdinalIgnoreCase))
{
throw;
}
}
var modelDeletionResult = dbClient.DeleteItemAsync(model.GetTable(), model.GetKey()).Result;
if (modelDeletionResult.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception("Failed to delete from classification table");
}
}
public override async void GridSearchAsync(ClassificationModel model)
{
var collection = await Classifier.GridSearchAsync(model);
if (collection.Contains("complexity"))
{
Complexity = collection["complexity"].Value;
}
if (collection.Contains("gamma"))
{
Gamma = collection["gamma"].Value;
}
if (collection.Contains("degree"))
{
Degree = (int)collection["degree"].Value;
}
}
// Fill
public string FillClassification(Classification Classification, ClassificationModel ClassificationModel, ContactOK contactOK)
{
Classification.DBCommand = (int)ClassificationModel.DBCommand;
Classification.ClassificationTVItemID = ClassificationModel.ClassificationTVItemID;
Classification.ClassificationType = (int)ClassificationModel.ClassificationType;
Classification.Ordinal = ClassificationModel.Ordinal;
Classification.LastUpdateDate_UTC = DateTime.UtcNow;
if (contactOK == null)
{
Classification.LastUpdateContactTVItemID = 2;
}
else
{
Classification.LastUpdateContactTVItemID = contactOK.ContactTVItemID;
}
return("");
}
public async Task <IndexResult> Index(string id, ClassificationModel existing)
{
var zipFile = await NgaDataAccess.GetHighResImageZipFile(id);
if (zipFile == null)
{
return(null);
}
byte[] imageBytes;
using (MemoryStream zipFileStream = new MemoryStream(zipFile))
using (ZipArchive archive = new ZipArchive(zipFileStream))
{
ZipArchiveEntry imgArchive = archive.Entries
.Single(x => ImageExtensions.Any(
imgExt => x.FullName.EndsWith(imgExt, StringComparison.OrdinalIgnoreCase)));
using (var memoryStream = new MemoryStream())
using (var imgStream = imgArchive.Open())
{
imgStream.CopyTo(memoryStream);
imageBytes = memoryStream.ToArray();
}
if (!imgArchive.FullName.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase))
{
imageBytes = new IndexingHttpClient().ConvertToJpeg(imageBytes).Result;
}
}
var classification = new ClassificationModel
{
Source = Source,
PageId = id
};
SetMetaData(classification);
if (imageBytes == null)
{
return(null);
}
return(new IndexResult
{
Model = classification,
ImageJpeg = Image.Load <Rgba64>(imageBytes)
});
}
private async Task Process(ClassificationModel classification)
{
Console.WriteLine($"Re-processing source: {classification.Source} page id: {classification.PageId}");
if (classification.ModerationLabels != null && !classification.Nudity.HasValue)
{
classification.Nudity = classification.ModerationLabels.Any(x =>
x.Name.Contains("nudity", StringComparison.OrdinalIgnoreCase) ||
x.ParentName.Contains("nudity", StringComparison.OrdinalIgnoreCase)
);
}
classification.S3Bucket = Constants.IMAGES_BUCKET;
var objectImage = S3Client.GetObjectAsync(Constants.IMAGES_BUCKET, $"{classification.S3Path}").Result;
if (objectImage.ContentLength == 0)
{
new ReviewProcess().MoveForReview(DbClient, ElasticSearchClient, S3Client, classification);
return;
}
byte[] imageBytes;
await using (var stream = objectImage.ResponseStream)
await using (var memoryStream = new MemoryStream())
{
await stream.CopyToAsync(memoryStream);
imageBytes = memoryStream.ToArray();
}
using var image = Image.Load(imageBytes);
classification.Height = image.Height;
classification.Width = image.Width;
classification.Orientation = image.Height >= image.Width ? Constants.ORIENTATION_PORTRAIT : Constants.ORIENTATION_LANDSCAPE;
var json = JObject.FromObject(classification, new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore
});
await ElasticSearchClient.SendToElasticSearch(classification);
await DbClient.PutItemAsync(
new ClassificationModel().GetTable(),
Document.FromJson(json.ToString()).ToAttributeMap()
);
}
private static void OpenAndParseCsv(List <ItemModel> itemModels, List <ProductClassificationSystemModel> classificationSystems)
{
using (var reader = new StreamReader(PATH_TO_CSV))
{
var header = reader.ReadLine();
var headerValues = header.Split(",");
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(',');
var classifications = new List <ClassificationModel>();
for (int i = 5; i < values.Count() - 1; i++)
{
var ps = classificationSystems.Where(cs => cs.countries.Any(p => p.country == headerValues[i])).FirstOrDefault();
ClassificationModel classification = new ClassificationModel()
{
systemCode = ps.systemCode, productCode = values[i]
};
if (classifications.Any(cs => cs.systemCode == classification.systemCode))
{
continue;
}
classifications.Add(classification);
}
ItemModel item = new ItemModel()
{
itemCode = values[0],
description = values[1],
companyId = COMPANY_ID,
classifications = classifications
};
itemModels.Add(item);
}
}
}
public RuntimeClassificationModel(ClassificationModel model)
{
FreeCoefficient = model.FreeCoefficient;
var rangeEntries = model.LimitPoints
.OrderBy(p => p)
.Select((p, i) => new RangeEntry(p.Point, i + 1));
ResutClassificator = new RangedListScaler(rangeEntries);
Properties = model.Properties
.Select(p => new RuntimeModelProperty
{
Name = p.Name,
Scaler = p.Entries.Any()
? new RangedListScaler(p.Entries.Select(e => new RangeEntry(e.LowerBound, e.Value)))
: (IScaler) new IdentityScaler(),
ModelCoefficient = p.ModelCoefficient
})
.ToList();
}
public static ClassificationModel ParseHtmlToNewModel(HtmlDocument htmlDoc, ClassificationModel model)
{
var infoDetails = htmlDoc.DocumentNode
.SelectNodes("//div[@class='h3_notice']")
?.Select(x => x.InnerText).ToList() ?? new List <string>();
var artistName = htmlDoc.DocumentNode
.SelectNodes("//h2")
?.Select(x => x.InnerText).ToList().FirstOrDefault() ?? string.Empty;
model.OriginalArtist = artistName.Trim();
model.Artist = Classifier.NormalizeArtist(artistName.Trim());
if (infoDetails.Count > 0)
{
model.Name = infoDetails[0].Trim();
}
if (infoDetails.Count > 1)
{
model.Date = infoDetails[1].Trim();
}
return(model);
}
/// <summary>
/// Generate a multi-class classification model using a specialist classifier for each class label.
/// </summary>
/// <param name="generator">The generator to use for each individual classifier.</param>
/// <param name="examples">Training examples of any number of classes</param>
/// <param name="trainingPercentage">Percentage of training examples to use, i.e. 70% = 0.7</param>
/// <param name="mixingPercentage">Percentage to mix positive and negative exmaples, i.e. 50% will add an additional 50% of
/// <paramref name="trainingPercentage"/> of negative examples into each classifier when training.</param>
/// <param name="isMultiClass">Determines whether each class is mutually inclusive.
/// <para>For example: If True, each class takes on a number of classes and does not necessarily belong to one specific class.</para>
/// <para>The ouput would then be a number of predicted classes for a single prediction. E.g. A song would be True as it may belong to classes: vocals, rock as well as bass.</para>
/// </param>
/// <returns></returns>
public static ClassificationModel Learn(IGenerator generator, IEnumerable<object> examples, double trainingPercentage, double mixingPercentage = 0.5, bool isMultiClass = true)
{
Descriptor descriptor = generator.Descriptor;
trainingPercentage = (trainingPercentage > 1.0 ? trainingPercentage / 100 : trainingPercentage);
mixingPercentage = (mixingPercentage > 1.0 ? mixingPercentage / 100 : mixingPercentage);
var classGroups = examples.Select(s => new
{
Label = generator.Descriptor.GetValue(s, descriptor.Label),
Item = s
})
.GroupBy(g => g.Label)
.ToDictionary(k => k.Key, v => v.Select(s => s.Item).ToArray());
int classes = classGroups.Count();
Dictionary<object, IClassifier> models = null;
Score finalScore = new Score();
if (classes > 2)
{
models = new Dictionary<object, IClassifier>(classes);
Task<Tuple<IClassifier, Score, object>>[] learningTasks = new Task<Tuple<IClassifier, Score, object>>[classes];
for (int y = 0; y < classes; y++)
{
models.Add(classGroups.ElementAt(y).Key, null);
int mix = (int)System.Math.Ceiling(((classGroups.ElementAt(y).Value.Count() * trainingPercentage) * mixingPercentage) / classes);
object label = classGroups.ElementAt(y).Key;
object[] truthExamples = classGroups.ElementAt(y).Value;
object[] falseExamples = classGroups.Where(w => w.Key != classGroups.Keys.ElementAt(y))
.SelectMany(s => s.Value.Take(mix).ToArray())
.ToArray();
learningTasks[y] = Task.Factory.StartNew(
() => MultiClassLearner.GenerateModel(generator, truthExamples, falseExamples, label, trainingPercentage, label)
);
}
Task.WaitAll(learningTasks);
Score[] scores = new Score[learningTasks.Count()];
for (int c = 0; c < learningTasks.Count(); c++)
{
models[learningTasks[c].Result.Item3] = learningTasks[c].Result.Item1;
scores[c] = learningTasks[c].Result.Item2;
}
finalScore = Score.CombineScores(scores);
}
else
{
// fallback to single classifier for two class classification
var dataset = descriptor.Convert(examples, true).ToExamples();
var positives = examples.Slice(dataset.Item2.Indices(f => f == 1d)).ToArray();
var negatives = examples.Slice(dataset.Item2.Indices(w => w != 1d)).ToArray();
var label = generator.Descriptor.GetValue(positives.First(), descriptor.Label);
var model = MultiClassLearner.GenerateModel(generator, positives, negatives, label, trainingPercentage, label);
finalScore = model.Item2;
models = new Dictionary<object, IClassifier>() { { label, model.Item1 } };
}
ClassificationModel classificationModel = new ClassificationModel()
{
Generator = generator,
Classifiers = models,
IsMultiClass = isMultiClass,
Score = finalScore
};
return classificationModel;
}