public static void ExtractForPlotlyTables(string fid)
{
Console.WriteLine(DateTime.Now);
DataFeatureExtractor.InitializeMetadata("en");
Console.WriteLine($"After InitializeMetadata: {DateTime.Now}");
PlotlyTable plotlyTable = PlotlyTable.GetInstance(fid);
List <PlotlyChart> plotlyCharts = PlotlyChart.GetInstance(fid, plotlyTable);
Console.WriteLine($"After GetInstance: {DateTime.Now}");
Common.FileInfo plotlyTableInfo = DataFeatureExtractor.ExtractTableFeatures(plotlyTable,
out List <Dictionary <int, Dictionary <string, float[]> > > headerEmbeddings,
out SourceFeatures sf);
Console.WriteLine($"After FE: {DateTime.Now}");
string uid = plotlyTable.TUid.Substring(0, plotlyTable.TUid.Length - 3);
Helpers.DumpJson($"{plotlyTable.TUid}.DF.json", sf, DataSerializer.Instance);
Helpers.DumpJson($"{plotlyTable.TUid}.table.json", plotlyTable, DataSerializer.Instance);
Helpers.DumpJson($"{uid}.EMB.json", headerEmbeddings, DataSerializer.Instance);
Helpers.DumpJson($"{uid}.index.json", plotlyTableInfo, DataSerializer.Instance);
Helpers.DumpCsv($"{plotlyTable.TUid}.csv", plotlyTable);
for (int i = 0; i < plotlyCharts.Count; i++)
{
PlotlyChart plotlyChart = plotlyCharts[i];
Helpers.DumpJson($"{plotlyChart.CUid}.json", plotlyChart, DataSerializer.Instance);
}
Console.WriteLine($"After Dump: {DateTime.Now}");
}
public static void ExtractForPlotlyTablesAll(string plotlyTableFolder)
{
Console.WriteLine($"Extracting PlotlyTable features from {plotlyTableFolder}.");
PlotlyList plotlyList = LoadPlotlyTablesAll(plotlyTableFolder);
List <PlotlyTable> plotlyTables = plotlyList.PlotlyTables;
List <PlotlyChart> plotlyCharts = plotlyList.PlotlyCharts;
// Initialize some models from MetadataRecoSvr to support the two Bayesian features.
DataFeatureExtractor.InitializeMetadata("en");
// Run features for each table and store embeddings for each plotly table.
for (int i = 0; i < plotlyTables.Count; i++)
{
PlotlyTable plotlyTable = plotlyTables[i];
Common.FileInfo plotlyTableInfo = DataFeatureExtractor.ExtractTableFeatures(plotlyTable,
out List <Dictionary <int, Dictionary <string, float[]> > > headerEmbeddings,
out SourceFeatures sf);
string uid = plotlyTable.TUid.Substring(0, plotlyTable.TUid.Length - 3);
Helpers.DumpJson($"{plotlyTable.TUid}.DF.json", sf, DataSerializer.Instance);
Helpers.DumpJson($"{plotlyTable.TUid}.table.json", plotlyTable, DataSerializer.Instance);
Helpers.DumpJson($"{uid}.EMB.json", headerEmbeddings, DataSerializer.Instance);
Helpers.DumpJson($"{uid}.index.json", plotlyTableInfo, DataSerializer.Instance);
Helpers.DumpCsv($"{plotlyTable.TUid}.csv", plotlyTable);
}
for (int i = 0; i < plotlyCharts.Count; i++)
{
PlotlyChart plotlyChart = plotlyCharts[i];
Helpers.DumpJson($"{plotlyChart.CUid}.json", plotlyChart, DataSerializer.Instance);
}
}
public static string GetFolder(string filePath)
{
FileInfo file = new FileInfo(filePath);
return(file.Directory.FullName);
}
private void _SearchFiles(SearchResult resultContainer)
{
Debug.Assert(resultContainer != null);
DatabaseAccess.MatchFilters activeFilter = m_activeSearchFilter != null ? m_activeSearchFilter.Type : DatabaseAccess.MatchFilters.Any;
if (resultContainer.SearchTerm == null || resultContainer.SearchTerm.Length <= 1)
{
return;
}
List <RawFileData> rawResults = new List <RawFileData>();
List <DBPredicate> searchPredicates = new List <DBPredicate>();
List <Common.FileInfo> convertedData = new List <Common.FileInfo>();
try
{
switch (activeFilter)
{
case DatabaseAccess.MatchFilters.Any:
{
if (resultContainer.WildcardSearch == true)
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
searchPredicates.Add(new DBPredicate("tags", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
}
else
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
searchPredicates.Add(new DBPredicate("tags", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
}
rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.OR);
break;
}
case DatabaseAccess.MatchFilters.Files:
{
if (resultContainer.WildcardSearch == true)
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
}
else
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
}
searchPredicates.Add(new DBPredicate("type", DBOperator.EQUALS, "1"));
rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.AND);
break;
}
case DatabaseAccess.MatchFilters.Folders:
{
if (resultContainer.WildcardSearch == true)
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
}
else
{
searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
}
searchPredicates.Add(new DBPredicate("type", DBOperator.EQUALS, "0"));
rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.AND);
break;
}
case DatabaseAccess.MatchFilters.Tags:
{
break;
}
default: break;
}
}
catch (Exception crap)
{
m_settings.SessionLog += crap.Message + "\n";
return;
}
if (rawResults.Count == 0)
{
return;
}
//take the retrieved raw data and convert it into the proper form
for (int i = rawResults.Count - 1; i >= 0; i--)
{
Common.FileInfo info = new Common.FileInfo(rawResults[i], Properties.Resources.file, Properties.Resources.folder);
info.CalculateWeight(resultContainer.SearchTerm);
convertedData.Add(info);
rawResults.RemoveAt(i);
}
/*sort the results in descending order based on relevancy weight*/
convertedData.Sort((a, b) => - 1 * a.Weight.CompareTo(b.Weight));
//remove all invalid files from the results list and store the results in the container
resultContainer.InvalidResults = _ValidateFiles(convertedData);
resultContainer.ValidResults = convertedData;
}
private void _OnFileChanged(object sender, FilePropertyChangedEventArgs e)
{
Common.FileInfo source = sender as Common.FileInfo;
if (source == null)
{
return;
}
try
{
//update the file record in the database
List <DBPredicate> updateValues = new List <DBPredicate>();
updateValues.Add(new DBPredicate("access_count", DBOperator.EQUALS, source.AccessCount.ToString()));
updateValues.Add(new DBPredicate("favorite", DBOperator.EQUALS, source.IsFavorite ? "1" : "0"));
updateValues.Add(new DBPredicate("hidden", DBOperator.EQUALS, source.IsHidden ? "1" : "0"));
m_database.Update("FileCache", updateValues, new DBPredicate("id", DBOperator.EQUALS, source.ID.ToString()));
}
catch (Exception crap)
{
Settings.SessionLog += crap.Message + "\n";
Debug.WriteLine(crap.Message);
}
if (LatestSearchResult == null)
{
return;
}
/*re-calculate the weight of the file and apply the changes by re-sorting the search result list*/
double initialWeight = source.Weight;
source.CalculateWeight(LatestSearchResult.SearchTerm);
if (initialWeight != source.Weight && Settings.SortImmediatelyOnFileChange && m_limitedResultCollection.Count > 1)
{
try
{
//if the first element was promoted, or the last element demoted no sorting is neccessary
if ((initialWeight < source.Weight && m_limitedResultCollection[0].SearchResultContent == source) ||
(initialWeight > source.Weight && m_limitedResultCollection[m_limitedResultCollection.Count - 1].SearchResultContent == source))
{
return;
}
int newIndex = -1;
int oldIndex = -1;
for (int i = 0; i < m_limitedResultCollection.Count; i++)
{
if (m_limitedResultCollection[i].SearchResultContent == source)
{
oldIndex = i;
if (newIndex != -1)
{
break;
}
}
else if (newIndex == -1)
{
if ((m_limitedResultCollection[i].SearchResultContent.Weight <= source.Weight))
{
newIndex = i;
if (oldIndex != -1)
{
break;
}
}
}
}
if (oldIndex == -1)
{
return;
}
if (newIndex == -1)
{
newIndex = m_limitedResultCollection.Count - 1;
}
//handle case where first item was demoted but is still the largest element (normally it would be swapped with the second element)
if ((oldIndex != 0 || newIndex != 1) || (m_limitedResultCollection[newIndex].SearchResultContent.Weight >= m_limitedResultCollection[oldIndex].SearchResultContent.Weight))
{
m_limitedResultCollection.Move(oldIndex, newIndex);
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}
}