public IActionResult redoElastic()
{
ElasticManager em = new ElasticManager();
em.fill();
return(Content("redone"));
}
public IActionResult Search(string search)
{
ElasticManager em = new ElasticManager();
var results = em.search(search);
return(View(results));
}
public IActionResult docdatasave(Document data)
{
using (var db = new ClassroomContext())
{
db.Update(data);
db.SaveChanges();
ElasticManager em = new ElasticManager();
em.addDocument(data);
}
return(Content("success"));
}
/// <summary>
/// Initializating managers and uOw
/// </summary>
public TaskExecuter()
{
uOw = new UnitOfWork();
elasticuOw = new ElasticUnitOfWork();
parsermanager = new ParserTaskManager(uOw);
propmanager = new PropertyManager(uOw);
goodManager = new GoodManager(uOw);
urlManager = new URLManager(uOw);
htmlValidator = new HtmlValidator();
priceManager = new PriceManager(uOw);
elasticManager = new ElasticManager(elasticuOw);
goodwizardManager = new GoodDatabasesWizard(elasticuOw, uOw);
taskinfoManager = new ExecuteManager(uOw);
}
public bool MainRoutine()
{
try
{
//Load File System Service
FileSystemManager fileMgr = new FileSystemManager();
NewElasticService.ServiceWCFClient svc = new ServiceWCFClient();
bool delete = svc.deleteAllRecommendedJob();
JobManager jobMgr = new JobManager();
String[] job_list = jobMgr.selectExpressionNames(); //job_names
double[] X = jobMgr.selectExpressionDifficulty(); //X
double[,] new_X = new double[X.Length, 1];
for (int i = 0; i < X.Length; i++)
{
new_X[i, 0] = X[i];
}
//User Profile (just the user ID, I still need the user self rating)
RecruiteeManager recMgr = new RecruiteeManager();
String[] recruitee_names = recMgr.selectRecruiteeNames();
double[] recruitee_skill = recMgr.selectRecruiteeSkills();
UserProfile[] users_profile = new UserProfile[recruitee_skill.Length];
for (int i = 0; i < recruitee_skill.Length; i++)
{
users_profile[i] = new UserProfile("", 0);
users_profile[i].UserID = recruitee_names[i];
users_profile[i].UserRating = recruitee_skill[i];
}
//new_Y
ElasticManager elaMgr = new ElasticManager();
double[,] Y = elaMgr.selectRatings(job_list, users_profile);
/////// WRITING VARIABLES IN FILE ////////////
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
////used to insert recommended jobs for a user in the database
fileMgr.writeFiles(job_list, new_X, users_profile, Y);
}).Start();
//Object to Hold Task Parameters
TaskDimensions task = new TaskDimensions();
task.num_features = new_X.GetLength(1); //1 is the number of columns
task.num_jobs_init = job_list.Length;
task.num_users_init = recruitee_names.Length;
//User number to start proccessing
int user_number = 1;
//Creating a variable to write in a File the job recommendations and comparisons
//Load File Writer
StreamWriter writeTextResult = fileMgr.getResultStreamWriter();
StreamWriter writeTextAverages = fileMgr.getAverageStreamWriter();
StreamWriter writeText = fileMgr.getIdandAvgStreamWriter();
StreamWriter writeTextDiff = fileMgr.getDifficultyStreamWriter();
double[] users_calculated_raitings = new double[task.num_users_init];
double total_rating_avg_system = 0;
double total_similarity_avg_system = 0;
double total_inaccuracy_system = 0;
MatlabManager matlabMgr = new MatlabManager();
while (user_number <= task.num_users_init)
{
double[,] my_ratings = new double[task.num_jobs_init, 1];
double[,] new_Y = new double[task.num_jobs_init, task.num_users_init - 1];
double[,] R = new double[task.num_jobs_init, task.num_users_init - 1];
for (int i = 0; i < job_list.Length; i++)
{
int k = 0;
for (int n = 0; n < users_profile.Length; n++)
{
if (n != (user_number - 1))
{
new_Y[i, k] = Y[i, n];
if (Y[i, n] != 0)
R[i, k] = 1;
else
R[i, k] = 0;
k++;
}
else
my_ratings[i, 0] = Y[i, n];
}
}
//Creating a MatLab reference to execute the recommended job script
object[] res = matlabMgr.executeFilter(task, job_list, Directory.GetCurrentDirectory() + DirectoryPaths.MATLAB, my_ratings, new_Y, R, new_X);
//Each time creates a to be used to write the recommended jobs in a file
List<TopJobData> mylist = fileMgr.writeValuesToFile(writeTextResult, res, job_list, user_number, new_X);
//Calculate Averages for Jobs for a User
DataResult avgs = new DataResult(mylist, mylist.Count, users_profile[user_number - 1]);
avgs.AverageForEachJob();
fileMgr.writeAveragesToFile(avgs, writeTextAverages, users_profile[user_number - 1]);
total_rating_avg_system += avgs.Rating_total_avg;
total_similarity_avg_system += avgs.Percentage_total_avg;
total_inaccuracy_system += avgs.Self_inaccuracy;
//adding the list at the Dictionary for each user
//ID and AVGs file
writeText.WriteLine(users_profile[user_number - 1].UserID + "\t" + avgs.Rating_total_avg);
users_calculated_raitings[user_number - 1] = avgs.Rating_total_avg;
//writing in the difficulty file
fileMgr.writeDifficultyToFile(writeTextDiff, avgs);
//used to insert recommended jobs for a user in the database
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
////used to insert recommended jobs for a user in the database
bool result = elaMgr.insertRecommenderJob(avgs);
}).Start();
user_number++;
}
total_rating_avg_system /= task.num_users_init;
total_similarity_avg_system /= task.num_users_init;
total_inaccuracy_system /= task.num_users_init;
//writing some more global information
fileMgr.writeGlobalAveragesInformation(total_rating_avg_system, total_similarity_avg_system, total_inaccuracy_system,
task, writeTextAverages, users_profile, users_calculated_raitings);
//closing the three files
writeText.Close();
writeTextResult.Close();
writeTextAverages.Close();
writeTextDiff.Close();
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
////used to insert recommended jobs for a user in the database
bool result = elaMgr.updateRanking(DirectoryPaths.FILE_ID_AVG);
}).Start();
Console.WriteLine("DONE");
//Wait until fisnih
Console.ReadLine();
return true;
}
catch (Exception ex)
{
return false;
}
}
public IActionResult UploadDocuments(IFormFile file)
{
Stream fileBuffer = file.OpenReadStream();
var db = new ClassroomContext();
Document doc = new Document();
doc.KeyName = WebUtility.UrlEncode(file.FileName);
doc.FileName = file.FileName;
if (file.ContentType.Contains("image"))
{
Stream fileBuffer2 = new MemoryStream();
fileBuffer.CopyTo(fileBuffer2);
fileBuffer.Seek(0, SeekOrigin.Begin);
IReadOnlyList <EntityAnnotation> resultText = GoogleMLConnector.ReadImageText(fileBuffer);
fileBuffer = fileBuffer2;
fileBuffer.Seek(0, SeekOrigin.Begin);
IReadOnlyList <EntityAnnotation> resultLabels = GoogleMLConnector.LabelImage(fileBuffer);
string lang = "";
string content = "";
foreach (var thing in resultText)
{
if (thing.Locale != null)
{
lang = thing.Locale;
}
content += thing.Description;
}
doc.Content = content;
content = "";
foreach (var thing in resultLabels)
{
content += thing.Description + ", ";
}
doc.GoogleTags = content;
fileBuffer = fileBuffer2;
}
else if (textValues.Contains <string>(file.ContentType))
{
StreamReader reader = new StreamReader(fileBuffer);
doc.Content = reader.ReadToEnd();
}
else if (file.ContentType.ToLower().Contains("pdf"))
{
string cont = "";
PdfFixedDocument pdfdoc = new PdfFixedDocument(fileBuffer);
List <PdfVisualImageCollection> img = new List <PdfVisualImageCollection>();
foreach (PdfPage page in pdfdoc.Pages)
{
PdfContentExtractor ce = new PdfContentExtractor(page);
cont += ce.ExtractText();
//img.Add(ce.ExtractImages(true));
}
doc.Content = cont;
}
else
{
StreamReader reader = new StreamReader(fileBuffer);
doc.Content = reader.ReadToEnd();
}
var uploadResult = S3Connector.UploadObject(file.FileName, fileBuffer, WebUtility.UrlEncode(file.FileName));
if (db.Documents.Where(x => x.KeyName == doc.KeyName).Count() == 0)
{
db.Documents.Add(doc);
db.SaveChanges();
ElasticManager em = new ElasticManager();
em.addDocument(doc);
}
return(Content(doc.Id.ToString()));
}
public IHttpActionResult Post([FromBody] string data)
{
string[] values = data.Split(',');
var searchword = values[0];
var from = int.Parse(values[1]);
var index = values[2];
var searchInBooks = values[3];
var _elastic = new ElasticManager();
var lemmas = new List <Lemma>();
var entries = new List <Entry>();
var accessFromLemma = new List <GetAccessData>();
var lemmaIds = new List <string>();
var totalHits = new long();
var suggestions = new List <string>();
#region ---First Search---
var lemmaResponse = _elastic.LemmaSearchByFullFormAndOrthography(searchword, from, index);
lemmaResponse.Hits?.ForEach(x => lemmas.Add(x.Source));
lemmas.ForEach(x => lemmaIds.Add(x.LemmaId));
//get entries for first search
if (lemmaResponse.Hits.Any())
{
var entryResponse = _elastic.EnrtrySearchInEntryByLemmaId(lemmaIds, index, searchInBooks);
entryResponse.Hits?.ForEach(y => entries.Add(y.Source));
totalHits = lemmaResponse.Total;
}
#endregion
#region ---Second and third Search---
else
{
var searchHeadWordExact = _elastic.EntrySearchByHeadWordExact(searchword, from, index, searchInBooks);
searchHeadWordExact.Hits?.ForEach(x => entries.Add(x.Source));
searchHeadWordExact.Hits?.ForEach(x => lemmaIds.Add(x.Source.EntryIdLemma.LemmaIdRef));
if (!searchHeadWordExact.Hits.Any())
{
//third search
var searchwordWild = "*" + searchword + "*";
//make 2 search - phrase and wildcard
var headWordWithPhrase = _elastic.EntrySearchByHeadWord(searchword, from, index, searchInBooks);
var headWordWithWildcard = _elastic.EntrySearchByHeadWordWithWildCard(searchwordWild, from, index,
searchInBooks);
//most hits win
if (headWordWithPhrase?.Total < headWordWithWildcard?.Total)
{
headWordWithWildcard?.Hits?.ForEach(x => entries.Add(x.Source));
headWordWithWildcard?.Hits?.ForEach(x => lemmaIds.Add(x.Source.EntryIdLemma.LemmaIdRef));
var first = headWordWithWildcard?.Suggest?.FirstOrDefault();
if (first != null)
{
var options = first.Value.Value.SelectMany(x => x.Options);
var suggestions2 = options?.Select(x => x.Text);
suggestions.AddRange(suggestions2);
}
}
else
{
headWordWithPhrase?.Hits?.ForEach(x => entries.Add(x.Source));
headWordWithPhrase?.Hits?.ForEach(x => lemmaIds.Add(x.Source.EntryIdLemma.LemmaIdRef));
var first = headWordWithPhrase?.Suggest?.FirstOrDefault();
if (first != null)
{
var options = first.Value.Value.SelectMany(x => x.Options);
var suggestions2 = options?.Select(x => x.Text);
suggestions.AddRange(suggestions2);
}
}
}
//getting lemmas from Entries:
var getLemmasFromEntries = _elastic.EnrtrySearchInLemmaByIdFromEntry(lemmaIds, index);
getLemmasFromEntries?.Hits?.ForEach(f => lemmas.Add(f.Source));
if (getLemmasFromEntries != null)
{
totalHits = getLemmasFromEntries.Total;
}
}
#endregion
#region ----getting info from accessoryData----
if (lemmas.Any())
{
foreach (var lemma in lemmas)
{
var lemmaFromAccess = new GetAccessData
{
WordOtrhography = lemma.LemmaOrtography,
SynonymsTo = new List <string>(),
StartsWith = new List <string>(),
EndsWith = new List <string>()
};
foreach (var accessData in lemma.LemmaAccessoryDatas)
{
switch (accessData.CategoryDan)
{
case "synonymer til ¤":
foreach (var refs in accessData.LemmaAccessDataReferencesRefs)
{
lemmaFromAccess.SynonymsTo.Add(refs.LemmaRef + ", " + refs.LemmaPos);
}
break;
case "ord der begynder med ¤":
foreach (var refs in accessData.LemmaAccessDataReferencesRefs)
{
lemmaFromAccess.StartsWith.Add(refs.LemmaRef + ", " + refs.LemmaPos);
}
break;
case "ord der ender på ¤":
foreach (var refs in accessData.LemmaAccessDataReferencesRefs)
{
lemmaFromAccess.EndsWith.Add(refs.LemmaRef + ", " + refs.LemmaPos);
}
break;
default:
Console.WriteLine("No AccessoryData found to " + lemma.LemmaOrtography);
break;
}
}
accessFromLemma.Add(lemmaFromAccess);
}
}
#endregion
//set search response
var searchResponse = new SearchResponse
{
Entries = entries,
Lemmas = lemmas,
GetAccessDatas = accessFromLemma,
TotalHits = totalHits,
Suggestions = suggestions
};
return(Ok(searchResponse));
}
public bool MainRoutine()
{
try
{
//Object to Hold Task Parameters
TaskDimensions task = new TaskDimensions();
//User number to start proccessing
int user_number = 1;
//Load File System Service
FileSystemManager fileMgr = new FileSystemManager();
//Method call to get the number of jobs and users from the file Y
fileMgr.detectSizeOfJobsColumns(task, Directory.GetCurrentDirectory() + DirectoryPaths.Y);
//Method call to get the number of features from the file X, and allocating the X matrix
double[,] X = fileMgr.getNumberOfFeaturesX(Directory.GetCurrentDirectory() + DirectoryPaths.X, task);
fileMgr.readFeaturesX(X, Directory.GetCurrentDirectory() + DirectoryPaths.X, task);
//Method call to get the jobs names
String[] job_list = fileMgr.readJobNames(Directory.GetCurrentDirectory() + DirectoryPaths.EXPRESSIONS, task);
//method that return the users profile
UserProfile[] users_profile = fileMgr.readUserProfile(Directory.GetCurrentDirectory() + DirectoryPaths.USER_TABLE, task);
//Creating a variable to write in a File the job recommendations and comparisons
//Load File Writer
StreamWriter writeTextResult = fileMgr.getResultStreamWriter();
StreamWriter writeTextAverages = fileMgr.getAverageStreamWriter();
StreamWriter writeText = fileMgr.getIdandAvgStreamWriter();
StreamWriter writeTextDiff = fileMgr.getDifficultyStreamWriter();
double[] users_calculated_raitings = new double[task.num_users_init];
double total_rating_avg_system = 0;
double total_similarity_avg_system = 0;
double total_inaccuracy_system = 0;
ElasticManager elaMgr = new ElasticManager();
while (user_number <= task.num_users_init)
{
// job rating file for a user
double[,] my_ratings = new double[task.num_jobs_init, 1];
//Now we read R and Y from theirs files (-1 because I will remove the chosen user from the matrixes)
double[,] Y = fileMgr.readTrainingY(Directory.GetCurrentDirectory() + DirectoryPaths.Y, task, my_ratings, user_number);
double[,] R = fileMgr.readTrainingR(Directory.GetCurrentDirectory() + DirectoryPaths.R, task, user_number);
//Creating a MatLab reference to execute the recommended job script
MatlabManager matlabMgr = new MatlabManager();
object[] res = matlabMgr.executeFilter(task, job_list, Directory.GetCurrentDirectory() + DirectoryPaths.MATLAB, my_ratings, Y, R, X);
//Each time creates a to be used to write the recommended jobs in a file
List<TopJobData> mylist = fileMgr.writeValuesToFile(writeTextResult, res, job_list, user_number, X);
//Calculate Averages for Jobs for a User
DataResult avgs = new DataResult(mylist, mylist.Count, users_profile[user_number - 1]);
avgs.AverageForEachJob();
fileMgr.writeAveragesToFile(avgs, writeTextAverages, users_profile[user_number - 1]);
total_rating_avg_system += avgs.Rating_total_avg;
total_similarity_avg_system += avgs.Percentage_total_avg;
total_inaccuracy_system += avgs.Self_inaccuracy;
//adding the list at the Dictionary for each user
//ID and AVGs file
writeText.WriteLine(users_profile[user_number - 1].UserID + "\t" + avgs.Rating_total_avg);
users_calculated_raitings[user_number - 1] = avgs.Rating_total_avg;
//writing in the difficulty file
fileMgr.writeDifficultyToFile(writeTextDiff, avgs);
//used to inC:\Users\larissaf\Desktop\FinaleVersionCrowd\recommenderSystems\Driver.cssert recommended jobs for a user in the database
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
////used to insert recommended jobs for a user in the database
bool result = elaMgr.insertRecommenderJob(avgs);
}).Start();
user_number++;
}
total_rating_avg_system /= task.num_users_init;
total_similarity_avg_system /= task.num_users_init;
total_inaccuracy_system /= task.num_users_init;
//writing some more global information
fileMgr.writeGlobalAveragesInformation(total_rating_avg_system, total_similarity_avg_system, total_inaccuracy_system,
task, writeTextAverages, users_profile, users_calculated_raitings);
//closing the three files
writeText.Close();
writeTextResult.Close();
writeTextAverages.Close();
writeTextDiff.Close();
/*
* Used to insert rating for task performed by workers. (User interface need to be built)
*
double[,] full_Y = svc.readFullY(Directory.GetCurrentDirectory() + "/files/Y.txt", task);
elaMgr.insertRatings(job_list, users_profile, full_Y);
*/
Console.WriteLine("DONE");
//Wait until fisnih
Console.ReadLine();
return true;
}
catch (Exception ex)
{
return false;
}
}