public override void PerformFunction(MySqlDataManipulator manipulator)
{
//Ensure that all KeywordPredictor models are loaded
//If one is not, then a company requesting that model through its settings will cause an error
if (!GlobalModelHelper.LoadOrTrainGlobalModels(ReflectionHelper.GetAllKeywordPredictors()))
{
throw new NullReferenceException("One or more global models failed to load. Server cannot start.");
}
DatabaseQueryProcessor processor = new DatabaseQueryProcessor(DatabaseQueryProcessorSettings.RetrieveCompanySettings(manipulator, CompanyId));
List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(CompanyId, "id > 0", validated: true);
List <string> sentences;
if (Flag.ToLower().Equals("complaint"))
{
//train model
sentences = validatedData.Select(entry => entry.Complaint).ToList();
if (!processor.TrainClusteringModels(manipulator, CompanyId, sentences, false))
{
Console.WriteLine("Failed to train problem prediction models for company " + CompanyId);
return;
}
//register the complaint groups that the clusterer predicts with the repair job entry in the database
foreach (RepairJobEntry entry in validatedData)
{
string groups = JsonDataObjectUtil <List <int> > .ConvertObject(processor.PredictGroupsInJobData(entry, CompanyId, manipulator));
entry.ComplaintGroups = groups;
manipulator.UpdateDataEntryGroups(CompanyId, entry, complaint: true);
}
}
Console.WriteLine("Trained clustering models for company " + CompanyId);
}
public static void PerformDataValidation(MySqlDataManipulator manipulator, int companyId, DatabaseQueryProcessor processor, int numShuffleTests = 5, int numGroups = 3)
{
List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id>0", validated: true);
List <RepairJobEntry> nonValidatedData = manipulator.GetDataEntriesWhere(companyId, "id>0", validated: false);
List <NonValidatedMapping> mappings = nonValidatedData.Select(entry => new NonValidatedMapping()
{
Entry = entry, Vote = 0
}).ToList();
double currCompanyAccuracy = manipulator.GetCompanyAccuracy(companyId);
for (int i = 0; i < numShuffleTests; i++)
{
mappings.Shuffle();
List <List <NonValidatedMapping> > nonValidatedTestingGroups = mappings.Split(numGroups);
foreach (List <NonValidatedMapping> currentTestGroup in nonValidatedTestingGroups)
{
List <RepairJobEntry> testGroup = new List <RepairJobEntry>(validatedData);
testGroup.AddRange(currentTestGroup.Select(mapping => mapping.Entry));
processor.TrainClusteringModels(manipulator, companyId, testGroup.Select(entry => entry.Complaint).ToList(), training: true);
double accuracy = 100 - PerformAutomatedTestingWithData(manipulator, companyId, processor, testGroup);
double vote = (accuracy - currCompanyAccuracy) / currCompanyAccuracy;
foreach (NonValidatedMapping mapping in currentTestGroup)
{
mapping.Vote += vote;
}
}
}
bool changed = false;
foreach (NonValidatedMapping mapping in mappings)
{
if (mapping.Vote > 0.01)
{
if (!manipulator.UpdateValidationStatus(companyId, mapping.Entry, wasValidated: false))
{
Console.WriteLine("Failed to update validation status of Repair Job Entry: " + mapping.Entry.Serialize(TableNameStorage.CompanyNonValidatedRepairJobTable.Replace("(n)", companyId.ToString())));
continue;
}
changed = true;
}
}
if (changed)
{
TrainClusteringModel(manipulator, processor, companyId, false);
}
}
public static void TrainClusteringModel(MySqlDataManipulator manipulator, DatabaseQueryProcessor processor, int companyId, bool training = false)
{
List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id > 0", validated: true);
List <string> sentences;
sentences = validatedData.Select(entry => entry.Complaint).ToList();
if (!processor.TrainClusteringModels(manipulator, companyId, sentences, training))
{
Console.WriteLine("Failed to train problem prediction models for company " + companyId);
return;
}
if (!training)
{
foreach (RepairJobEntry entry in validatedData)
{
string groups = JsonDataObjectUtil <List <int> > .ConvertObject(processor.PredictGroupsInJobData(entry, companyId, manipulator));
entry.ComplaintGroups = groups;
manipulator.UpdateDataEntryGroups(companyId, entry, complaint: true);
}
}
}
private void HandlePutRequest(HttpListenerContext ctx)
{
try
{
#region Input Validation
if (!ctx.Request.HasEntityBody)
{
WriteBodyResponse(ctx, 400, "Bad Request", "No Body");
return;
}
ArchiveApiPutRequest req = JsonDataObjectUtil <ArchiveApiPutRequest> .ParseObject(ctx);
if (!ValidatePutRequest(req))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Incorrect Format");
return;
}
#endregion
MySqlDataManipulator connection = new MySqlDataManipulator();
using (connection)
{
bool res = connection.Connect(MySqlDataManipulator.GlobalConfiguration.GetConnectionString());
if (!res)
{
WriteBodyResponse(ctx, 500, "Unexpected Server Error", "Connection to database failed");
return;
}
#region Validate User
OverallUser mappedUser = connection.GetUserById(req.UserId);
if (mappedUser == null)
{
WriteBodyResponse(ctx, 404, "Not Found", "User was not found on on the server");
return;
}
if (!UserVerificationUtil.LoginTokenValid(mappedUser, req.LoginToken))
{
WriteBodyResponse(ctx, 401, "Not Authorized", "Login token was incorrect.");
return;
}
CompanySettingsEntry isPublicSetting = connection.GetCompanySettingsWhere(req.CompanyId, "SettingKey=\"" + CompanySettingsKey.Public + "\"")[0];
bool isPublic = bool.Parse(isPublicSetting.SettingValue);
if (!isPublic && mappedUser.Company != req.CompanyId)
{
WriteBodyResponse(ctx, 401, "Not Authorized", "Cannot access other company's private data");
return;
}
#endregion
UserSettingsEntry numPredictionsRequested = JsonDataObjectUtil <List <UserSettingsEntry> > .ParseObject(mappedUser.Settings).FirstOrDefault(entry => entry.Key.Equals(UserSettingsEntryKeys.ArchiveQueryResults));
if (numPredictionsRequested == null)
{
WriteBodyResponse(ctx, 500, "Internal Server Error", "User did not contain a setting with a key " + UserSettingsEntryKeys.ArchiveQueryResults);
return;
}
int numRequested = int.Parse(numPredictionsRequested.Value);
#region Input sanitation
string whereString = "";
bool addedWhere = false;
if (req.Entry.Complaint != null)
{
if (!PerformSanitization(req.Entry.Complaint))
{
return;
}
whereString += " Complaint like \"%" + req.Entry.Complaint + "%\"";
addedWhere = true;
}
if (req.Entry.Problem != null)
{
if (!PerformSanitization(req.Entry.Problem))
{
return;
}
if (addedWhere)
{
whereString += " and";
}
whereString += " Problem like \"%" + req.Entry.Problem + "%\"";
addedWhere = true;
}
if (req.Entry.Make != null)
{
if (!PerformSanitization(req.Entry.Make))
{
return;
}
if (addedWhere)
{
whereString += " and";
}
whereString += " Make like \"%" + req.Entry.Make + "%\"";
addedWhere = true;
}
if (req.Entry.Model != null)
{
if (!PerformSanitization(req.Entry.Model))
{
return;
}
if (addedWhere)
{
whereString += " and";
}
whereString += " Model like \"%" + req.Entry.Model + "%\"";
addedWhere = true;
}
if (req.Entry.Year != 0)
{
if (addedWhere)
{
whereString += " and";
}
whereString += " Year =" + req.Entry.Year;
addedWhere = true;
}
#endregion
if (!addedWhere)
{
WriteBodyResponse(ctx, 400, "Bad Request", "No fields in the request's entry were filled");
return;
}
List <RepairJobEntry> entries = connection.GetDataEntriesWhere(req.CompanyId, whereString, true);
JsonListStringConstructor retConstructor = new JsonListStringConstructor();
try
{
entries.ForEach(entry => retConstructor.AddElement(ConvertEntry(entry)));
} catch (NullReferenceException)
{
WriteBodyResponse(ctx, 200, "OK", "[]", "application/json");
return;
}
WriteBodyResponse(ctx, 200, "OK", retConstructor.ToString(), "application/json");
bool PerformSanitization(string queryIn)
{
if (queryIn.Contains('`'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the single quote character, which is disallowed due to MySQL injection attacks");
return(false);
}
return(true);
}
}
}
catch (HttpListenerException)
{
//HttpListeners dispose themselves when an exception occurs, so we can do no more.
}
catch (Exception e)
{
WriteBodyResponse(ctx, 500, "Internal Server Error", "Error occurred during processing of request: " + e.Message);
}
}
public static double PerformAutomatedTesting(MySqlDataManipulator manipulator, int companyId, DatabaseQueryProcessor processor)
{
List <RepairJobEntry> validatedData = manipulator.GetDataEntriesWhere(companyId, "id > 0", validated: true);
return(PerformAutomatedTestingWithData(manipulator, companyId, processor, validatedData));
}
/// <summary>
/// Request for adding a repair job entry. Documention is found in the Web API Enumeration file
/// in the /RepairJob tab, starting at row 1
/// </summary>
/// <param name="ctx">The HttpListenerContext to respond to</param>
private void HandlePostRequest(HttpListenerContext ctx)
{
try
{
#region Input Validation
if (!ctx.Request.HasEntityBody)
{
WriteBodyResponse(ctx, 400, "Bad Request", "No Body");
return;
}
RepairJobApiRequest entry = JsonDataObjectUtil <RepairJobApiRequest> .ParseObject(ctx);
if (!ValidateFullRequest(entry))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Incorrect Format");
return;
}
#endregion
//Otherwise we have a valid entry, validate user
MySqlDataManipulator connection = new MySqlDataManipulator();
using (connection)
{
bool res = connection.Connect(MySqlDataManipulator.GlobalConfiguration.GetConnectionString());
if (!res)
{
WriteBodyResponse(ctx, 500, "Unexpected ServerError", "Connection to database failed");
return;
}
#region User Validation
OverallUser mappedUser = connection.GetUserById(entry.UserId);
if (mappedUser == null)
{
WriteBodyResponse(ctx, 404, "Not Found", "User was not found on the server");
return;
}
if (!UserVerificationUtil.LoginTokenValid(mappedUser, entry.LoginToken))
{
WriteBodyResponse(ctx, 401, "Not Authorized", "Login token was incorrect.");
return;
}
if (!UserVerificationUtil.AuthTokenValid(mappedUser, entry.AuthToken))
{
WriteBodyResponse(ctx, 401, "Not Authorized", "Auth token was expired or incorrect");
return;
}
#endregion
#region Input Sanitation
if (entry.ContainedEntry.Complaint.Contains('<'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks");
return;
}
if (entry.ContainedEntry.Problem.Contains('<'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks");
return;
}
if (entry.ContainedEntry.Make.Contains('<'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks");
return;
}
if (entry.ContainedEntry.Model.Contains('<'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks");
return;
}
if (entry.ContainedEntry.JobId.Contains('<'))
{
WriteBodyResponse(ctx, 400, "Bad Request", "Request contained the < character, which is disallowed due to cross site scripting attacks");
return;
}
#endregion
#region Action Handling
#region Forced Upload
if (!(entry.Duplicate == 0))
{
//Now that we know the user is good, actually do the addition.
res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry);
if (!res)
{
WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message);
return;
}
WriteBodylessResponse(ctx, 200, "OK");
}
#endregion
else
{
//test if there exists similar
string whereString = "Make =\"" + entry.ContainedEntry.Make + "\" AND " + "Model =\"" + entry.ContainedEntry.Model + "\"";
//whereString += "AND"+entry.ContainedEntry.Year+">="+(entry.ContainedEntry.Year-2)+"AND"+entry.ContainedEntry.Year+"<="+(entry.ContainedEntry.Year+2);
List <RepairJobEntry> dataCollectionsWhere = connection.GetDataEntriesWhere(mappedUser.Company, whereString, true);
List <RepairJobEntry> data2 = connection.GetDataEntriesWhere(mappedUser.Company, whereString, false);
foreach (RepairJobEntry x in data2)
{
dataCollectionsWhere.Add(x);
}
#region No Similar Jobs
//if none force through
if (dataCollectionsWhere.Count == 0)
{
res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry);
if (!res)
{
WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message);
return;
}
WriteBodylessResponse(ctx, 200, "OK");
}
#endregion
#region Similar Jobs Return
//if yes 409 with similar jobs
else
{
JsonListStringConstructor retConstructor = new JsonListStringConstructor();
List <EntrySimilarity> ret = getSimilar(entry.ContainedEntry, dataCollectionsWhere, 3);
if (ret.Count == 0)
{
res = connection.AddDataEntry(mappedUser.Company, entry.ContainedEntry);
if (!res)
{
WriteBodyResponse(ctx, 500, "Unexpected Server Error", connection.LastException.Message);
return;
}
WriteBodylessResponse(ctx, 200, "OK");
}
ret.ForEach(obj => retConstructor.AddElement(ConvertEntrySimilarity(obj)));
WriteBodyResponse(ctx, 409, "Conflict", retConstructor.ToString(), "application/json");
JsonDictionaryStringConstructor ConvertEntrySimilarity(EntrySimilarity e)
{
JsonDictionaryStringConstructor r = new JsonDictionaryStringConstructor();
r.SetMapping("Make", e.Entry.Make);
r.SetMapping("Model", e.Entry.Model);
r.SetMapping("Complaint", e.Entry.Complaint);
r.SetMapping("Problem", e.Entry.Problem);
if (e.Entry.Year == -1)
{
r.SetMapping("Year", "Unknown");
}
else
{
r.SetMapping("Year", e.Entry.Year);
}
r.SetMapping("Id", e.Entry.Id);
return(r);
}
}
#endregion
}
#endregion
}
}
catch (HttpListenerException)
{
//HttpListeners dispose themselves when an exception occurs, so we can do no more.
}
catch (Exception e)
{
WriteBodyResponse(ctx, 500, "Internal Server Error", e.Message);
}
}