public override void PerformFunction(MySqlDataManipulator manipulator) { //Ensure that all KeywordPredictor models are loaded //If one is not, then a company requesting that model through its settings will cause an error if (!GlobalModelHelper.LoadOrTrainGlobalModels(ReflectionHelper.GetAllKeywordPredictors())) { throw new NullReferenceException("One or more global models failed to load. Server cannot start."); } DatabaseQueryProcessor processor = new DatabaseQueryProcessor(DatabaseQueryProcessorSettings.RetrieveCompanySettings(manipulator, CompanyId)); List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(CompanyId, "id > 0", validated: true); List <string> sentences; if (Flag.ToLower().Equals("complaint")) { //train model sentences = validatedData.Select(entry => entry.Complaint).ToList(); if (!processor.TrainClusteringModels(manipulator, CompanyId, sentences, false)) { Console.WriteLine("Failed to train problem prediction models for company " + CompanyId); return; } //register the complaint groups that the clusterer predicts with the repair job entry in the database foreach (RepairJobEntry entry in validatedData) { string groups = JsonDataObjectUtil <List <int> > .ConvertObject(processor.PredictGroupsInJobData(entry, CompanyId, manipulator)); entry.ComplaintGroups = groups; manipulator.UpdateDataEntryGroups(CompanyId, entry, complaint: true); } } Console.WriteLine("Trained clustering models for company " + CompanyId); }
public static void PerformDataValidation(MySqlDataManipulator manipulator, int companyId, DatabaseQueryProcessor processor, int numShuffleTests = 5, int numGroups = 3) { List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id>0", validated: true); List <RepairJobEntry> nonValidatedData = manipulator.GetDataEntriesWhere(companyId, "id>0", validated: false); List <NonValidatedMapping> mappings = nonValidatedData.Select(entry => new NonValidatedMapping() { Entry = entry, Vote = 0 }).ToList(); double currCompanyAccuracy = manipulator.GetCompanyAccuracy(companyId); for (int i = 0; i < numShuffleTests; i++) { mappings.Shuffle(); List <List <NonValidatedMapping> > nonValidatedTestingGroups = mappings.Split(numGroups); foreach (List <NonValidatedMapping> currentTestGroup in nonValidatedTestingGroups) { List <RepairJobEntry> testGroup = new List <RepairJobEntry>(validatedData); testGroup.AddRange(currentTestGroup.Select(mapping => mapping.Entry)); processor.TrainClusteringModels(manipulator, companyId, testGroup.Select(entry => entry.Complaint).ToList(), training: true); double accuracy = 100 - PerformAutomatedTestingWithData(manipulator, companyId, processor, testGroup); double vote = (accuracy - currCompanyAccuracy) / currCompanyAccuracy; foreach (NonValidatedMapping mapping in currentTestGroup) { mapping.Vote += vote; } } } bool changed = false; foreach (NonValidatedMapping mapping in mappings) { if (mapping.Vote > 0.01) { if (!manipulator.UpdateValidationStatus(companyId, mapping.Entry, wasValidated: false)) { Console.WriteLine("Failed to update validation status of Repair Job Entry: " + mapping.Entry.Serialize(TableNameStorage.CompanyNonValidatedRepairJobTable.Replace("(n)", companyId.ToString()))); continue; } changed = true; } } if (changed) { TrainClusteringModel(manipulator, processor, companyId, false); } }
public static void TrainClusteringModel(MySqlDataManipulator manipulator, DatabaseQueryProcessor processor, int companyId, bool training = false) { List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id > 0", validated: true); List <string> sentences; sentences = validatedData.Select(entry => entry.Complaint).ToList(); if (!processor.TrainClusteringModels(manipulator, companyId, sentences, training)) { Console.WriteLine("Failed to train problem prediction models for company " + companyId); return; } if (!training) { foreach (RepairJobEntry entry in validatedData) { string groups = JsonDataObjectUtil <List <int> > .ConvertObject(processor.PredictGroupsInJobData(entry, companyId, manipulator)); entry.ComplaintGroups = groups; manipulator.UpdateDataEntryGroups(companyId, entry, complaint: true); } } }
private void HandlePutRequest(HttpListenerContext ctx) { try { #region Input Validation if (!ctx.Request.HasEntityBody) { WriteBodyResponse(ctx, 400, "Bad Request", "No Body"); return; } ArchiveApiPutRequest req = JsonDataObjectUtil <ArchiveApiPutRequest> .ParseObject(ctx); if (!ValidatePutRequest(req)) { WriteBodyResponse(ctx, 400, "Bad Request", "Incorrect Format"); return; } #endregion MySqlDataManipulator connection = new MySqlDataManipulator(); using (connection) { bool res = connection.Connect(MySqlDataManipulator.GlobalConfiguration.GetConnectionString()); if (!res) { WriteBodyResponse(ctx, 500, "Unexpected Server Error", "Connection to database failed"); return; } #region Validate User OverallUser mappedUser = connection.GetUserById(req.UserId); if (mappedUser == null) { WriteBodyResponse(ctx, 404, "Not Found", "User was not found on on the server"); return; } if (!UserVerificationUtil.LoginTokenValid(mappedUser, req.LoginToken)) { WriteBodyResponse(ctx, 401, "Not Authorized", "Login token was incorrect."); return; } CompanySettingsEntry isPublicSetting = connection.GetCompanySettingsWhere(req.CompanyId, "SettingKey=\"" + CompanySettingsKey.Public + "\"")[0]; bool isPublic = bool.Parse(isPublicSetting.SettingValue); if (!isPublic && mappedUser.Company != req.CompanyId) { WriteBodyResponse(ctx, 401, "Not Authorized", "Cannot access other company's private data"); return; } #endregion UserSettingsEntry numPredictionsRequested = JsonDataObjectUtil <List <UserSettingsEntry> > .ParseObject(mappedUser.Settings).FirstOrDefault(entry => entry.Key.Equals(UserSettingsEntryKeys.ArchiveQueryResults)); if (numPredictionsRequested == null) { WriteBodyResponse(ctx, 500, "Internal Server Error", "User did not contain a setting with a key " + UserSettingsEntryKeys.ArchiveQueryResults); return; } int numRequested = int.Parse(numPredictionsRequested.Value); #region Input sanitation string whereString = ""; bool addedWhere = false; if (req.Entry.Complaint != null) { if (!PerformSanitization(req.Entry.Complaint)) { return; } whereString += " Complaint like \"%" + req.Entry.Complaint + "%\""; addedWhere = true; } if (req.Entry.Problem != null) { if (!PerformSanitization(req.Entry.Problem)) { return; } if (addedWhere) { whereString += " and"; } whereString += " Problem like \"%" + req.Entry.Problem + "%\""; addedWhere = true; } if (req.Entry.Make != null) { if (!PerformSanitization(req.Entry.Make)) { return; } if (addedWhere) { whereString += " and"; } whereString += " Make like \"%" + req.Entry.Make + "%\""; addedWhere = true; } if (req.Entry.Model != null) { if (!PerformSanitization(req.Entry.Model)) { return; } if (addedWhere) { whereString += " and"; } whereString += " Model like \"%" + req.Entry.Model + "%\""; addedWhere = true; } if (req.Entry.Year != 0) { if (addedWhere) { whereString += " and"; } whereString += " Year =" + req.Entry.Year; addedWhere = true; } #endregion if (!addedWhere) { WriteBodyResponse(ctx, 400, "Bad Request", "No fields in the request's entry were filled"); return; } List <RepairJobEntry> entries = connection.GetDataEntriesWhere(req.CompanyId, whereString, true); JsonListStringConstructor retConstructor = new JsonListStringConstructor(); try { entries.ForEach(entry => retConstructor.AddElement(ConvertEntry(entry))); } catch (NullReferenceException) { WriteBodyResponse(ctx, 200, "OK", "[]", "application/json"); return; } WriteBodyResponse(ctx, 200, "OK", retConstructor.ToString(), "application/json"); bool PerformSanitization(string queryIn) { if (queryIn.Contains('`')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the single quote character, which is disallowed due to MySQL injection attacks"); return(false); } return(true); } } } catch (HttpListenerException) { //HttpListeners dispose themselves when an exception occurs, so we can do no more. } catch (Exception e) { WriteBodyResponse(ctx, 500, "Internal Server Error", "Error occurred during processing of request: " + e.Message); } }
public static double PerformAutomatedTesting(MySqlDataManipulator manipulator, int companyId, DatabaseQueryProcessor processor) { List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id > 0", validated: true); return(PerformAutomatedTestingWithData(manipulator, companyId, processor, validatedData)); }
/// <summary> /// Request for adding a repair job entry. Documention is found in the Web API Enumeration file /// in the /RepairJob tab, starting at row 1 /// </summary> /// <param name="ctx">The HttpListenerContext to respond to</param> private void HandlePostRequest(HttpListenerContext ctx) { try { #region Input Validation if (!ctx.Request.HasEntityBody) { WriteBodyResponse(ctx, 400, "Bad Request", "No Body"); return; } RepairJobApiRequest entry = JsonDataObjectUtil <RepairJobApiRequest> .ParseObject(ctx); if (!ValidateFullRequest(entry)) { WriteBodyResponse(ctx, 400, "Bad Request", "Incorrect Format"); return; } #endregion //Otherwise we have a valid entry, validate user MySqlDataManipulator connection = new MySqlDataManipulator(); using (connection) { bool res = connection.Connect(MySqlDataManipulator.GlobalConfiguration.GetConnectionString()); if (!res) { WriteBodyResponse(ctx, 500, "Unexpected ServerError", "Connection to database failed"); return; } #region User Validation OverallUser mappedUser = connection.GetUserById(entry.UserId); if (mappedUser == null) { WriteBodyResponse(ctx, 404, "Not Found", "User was not found on the server"); return; } if (!UserVerificationUtil.LoginTokenValid(mappedUser, entry.LoginToken)) { WriteBodyResponse(ctx, 401, "Not Authorized", "Login token was incorrect."); return; } if (!UserVerificationUtil.AuthTokenValid(mappedUser, entry.AuthToken)) { WriteBodyResponse(ctx, 401, "Not Authorized", "Auth token was expired or incorrect"); return; } #endregion #region Input Sanitation if (entry.ContainedEntry.Complaint.Contains('<')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks"); return; } if (entry.ContainedEntry.Problem.Contains('<')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks"); return; } if (entry.ContainedEntry.Make.Contains('<')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks"); return; } if (entry.ContainedEntry.Model.Contains('<')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks"); return; } if (entry.ContainedEntry.JobId.Contains('<')) { WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks"); return; } #endregion #region Action Handling #region Forced Upload if (!(entry.Duplicate == 0)) { //Now that we know the user is good, actually do the addition. res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry); if (!res) { WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message); return; } WriteBodylessResponse(ctx, 200, "OK"); } #endregion else { //test if there exists similar string whereString = "Make =\"" + entry.ContainedEntry.Make + "\" AND " + "Model =\"" + entry.ContainedEntry.Model + "\""; //whereString += "AND"+entry.ContainedEntry.Year+">="+(entry.ContainedEntry.Year-2)+"AND"+entry.ContainedEntry.Year+"<="+(entry.ContainedEntry.Year+2); List <RepairJobEntry> dataCollectionsWhere = connection.GetDataEntriesWhere(mappedUser.Company, whereString, true); List <RepairJobEntry> data2 = connection.GetDataEntriesWhere(mappedUser.Company, whereString, false); foreach (RepairJobEntry x in data2) { dataCollectionsWhere.Add(x); } #region No Similar Jobs //if none force through if (dataCollectionsWhere.Count == 0) { res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry); if (!res) { WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message); return; } WriteBodylessResponse(ctx, 200, "OK"); } #endregion #region Similar Jobs Return //if yes 409 with similar jobs else { JsonListStringConstructor retConstructor = new JsonListStringConstructor(); List <EntrySimilarity> ret = getSimilar(entry.ContainedEntry, dataCollectionsWhere, 3); if (ret.Count == 0) { res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry); if (!res) { WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message); return; } WriteBodylessResponse(ctx, 200, "OK"); } ret.ForEach(obj => retConstructor.AddElement(ConvertEntrySimilarity(obj))); WriteBodyResponse(ctx, 409, "Conflict", retConstructor.ToString(), "application/json"); JsonDictionaryStringConstructor ConvertEntrySimilarity(EntrySimilarity e) { JsonDictionaryStringConstructor r = new JsonDictionaryStringConstructor(); r.SetMapping("Make", e.Entry.Make); r.SetMapping("Model", e.Entry.Model); r.SetMapping("Complaint", e.Entry.Complaint); r.SetMapping("Problem", e.Entry.Problem); if (e.Entry.Year == -1) { r.SetMapping("Year", "Unknown"); } else { r.SetMapping("Year", e.Entry.Year); } r.SetMapping("Id", e.Entry.Id); return(r); } } #endregion } #endregion } } catch (HttpListenerException) { //HttpListeners dispose themselves when an exception occurs, so we can do no more. } catch (Exception e) { WriteBodyResponse(ctx, 500, "Internal Server Error", e.Message); } }