public static void ExtractForPlotlyTables(string fid)
        {
            Console.WriteLine(DateTime.Now);
            DataFeatureExtractor.InitializeMetadata("en");
            Console.WriteLine($"After InitializeMetadata: {DateTime.Now}");
            PlotlyTable        plotlyTable  = PlotlyTable.GetInstance(fid);
            List <PlotlyChart> plotlyCharts = PlotlyChart.GetInstance(fid, plotlyTable);

            Console.WriteLine($"After GetInstance: {DateTime.Now}");
            Common.FileInfo plotlyTableInfo = DataFeatureExtractor.ExtractTableFeatures(plotlyTable,
                                                                                        out List <Dictionary <int, Dictionary <string, float[]> > > headerEmbeddings,
                                                                                        out SourceFeatures sf);
            Console.WriteLine($"After FE: {DateTime.Now}");
            string uid = plotlyTable.TUid.Substring(0, plotlyTable.TUid.Length - 3);

            Helpers.DumpJson($"{plotlyTable.TUid}.DF.json", sf, DataSerializer.Instance);
            Helpers.DumpJson($"{plotlyTable.TUid}.table.json", plotlyTable, DataSerializer.Instance);
            Helpers.DumpJson($"{uid}.EMB.json", headerEmbeddings, DataSerializer.Instance);
            Helpers.DumpJson($"{uid}.index.json", plotlyTableInfo, DataSerializer.Instance);
            Helpers.DumpCsv($"{plotlyTable.TUid}.csv", plotlyTable);
            for (int i = 0; i < plotlyCharts.Count; i++)
            {
                PlotlyChart plotlyChart = plotlyCharts[i];
                Helpers.DumpJson($"{plotlyChart.CUid}.json", plotlyChart, DataSerializer.Instance);
            }
            Console.WriteLine($"After Dump: {DateTime.Now}");
        }
        public static void ExtractForPlotlyTablesAll(string plotlyTableFolder)
        {
            Console.WriteLine($"Extracting PlotlyTable features from {plotlyTableFolder}.");

            PlotlyList         plotlyList   = LoadPlotlyTablesAll(plotlyTableFolder);
            List <PlotlyTable> plotlyTables = plotlyList.PlotlyTables;
            List <PlotlyChart> plotlyCharts = plotlyList.PlotlyCharts;

            // Initialize some models from MetadataRecoSvr to support the two Bayesian features.
            DataFeatureExtractor.InitializeMetadata("en");

            // Run features for each table and store embeddings for each plotly table.
            for (int i = 0; i < plotlyTables.Count; i++)
            {
                PlotlyTable     plotlyTable     = plotlyTables[i];
                Common.FileInfo plotlyTableInfo = DataFeatureExtractor.ExtractTableFeatures(plotlyTable,
                                                                                            out List <Dictionary <int, Dictionary <string, float[]> > > headerEmbeddings,
                                                                                            out SourceFeatures sf);
                string uid = plotlyTable.TUid.Substring(0, plotlyTable.TUid.Length - 3);
                Helpers.DumpJson($"{plotlyTable.TUid}.DF.json", sf, DataSerializer.Instance);
                Helpers.DumpJson($"{plotlyTable.TUid}.table.json", plotlyTable, DataSerializer.Instance);
                Helpers.DumpJson($"{uid}.EMB.json", headerEmbeddings, DataSerializer.Instance);
                Helpers.DumpJson($"{uid}.index.json", plotlyTableInfo, DataSerializer.Instance);
                Helpers.DumpCsv($"{plotlyTable.TUid}.csv", plotlyTable);
            }
            for (int i = 0; i < plotlyCharts.Count; i++)
            {
                PlotlyChart plotlyChart = plotlyCharts[i];
                Helpers.DumpJson($"{plotlyChart.CUid}.json", plotlyChart, DataSerializer.Instance);
            }
        }
        public static string GetFolder(string filePath)
        {
            FileInfo file = new FileInfo(filePath);

            return(file.Directory.FullName);
        }
Пример #4
0
        private void _SearchFiles(SearchResult resultContainer)
        {
            Debug.Assert(resultContainer != null);

            DatabaseAccess.MatchFilters activeFilter = m_activeSearchFilter != null ? m_activeSearchFilter.Type : DatabaseAccess.MatchFilters.Any;

            if (resultContainer.SearchTerm == null || resultContainer.SearchTerm.Length <= 1)
            {
                return;
            }

            List <RawFileData>     rawResults       = new List <RawFileData>();
            List <DBPredicate>     searchPredicates = new List <DBPredicate>();
            List <Common.FileInfo> convertedData    = new List <Common.FileInfo>();

            try
            {
                switch (activeFilter)
                {
                case DatabaseAccess.MatchFilters.Any:
                {
                    if (resultContainer.WildcardSearch == true)
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
                        searchPredicates.Add(new DBPredicate("tags", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
                    }
                    else
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
                        searchPredicates.Add(new DBPredicate("tags", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
                    }
                    rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.OR);
                    break;
                }

                case DatabaseAccess.MatchFilters.Files:
                {
                    if (resultContainer.WildcardSearch == true)
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
                    }
                    else
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
                    }
                    searchPredicates.Add(new DBPredicate("type", DBOperator.EQUALS, "1"));

                    rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.AND);
                    break;
                }

                case DatabaseAccess.MatchFilters.Folders:
                {
                    if (resultContainer.WildcardSearch == true)
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, "%" + resultContainer.SearchTerm + "%"));
                    }
                    else
                    {
                        searchPredicates.Add(new DBPredicate("name", DBOperator.LIKE, resultContainer.SearchTerm + "%"));
                    }
                    searchPredicates.Add(new DBPredicate("type", DBOperator.EQUALS, "0"));

                    rawResults = m_database.Select <RawFileData>("FileCache", searchPredicates, new RawFileDataCreator(), DBConjunction.AND);
                    break;
                }

                case DatabaseAccess.MatchFilters.Tags:
                {
                    break;
                }

                default: break;
                }
            }
            catch (Exception crap)
            {
                m_settings.SessionLog += crap.Message + "\n";
                return;
            }

            if (rawResults.Count == 0)
            {
                return;
            }

            //take the retrieved raw data and convert it into the proper form
            for (int i = rawResults.Count - 1; i >= 0; i--)
            {
                Common.FileInfo info = new Common.FileInfo(rawResults[i], Properties.Resources.file, Properties.Resources.folder);
                info.CalculateWeight(resultContainer.SearchTerm);
                convertedData.Add(info);
                rawResults.RemoveAt(i);
            }

            /*sort the results in descending order based on relevancy weight*/
            convertedData.Sort((a, b) => - 1 * a.Weight.CompareTo(b.Weight));

            //remove all invalid files from the results list and store the results in the container
            resultContainer.InvalidResults = _ValidateFiles(convertedData);
            resultContainer.ValidResults   = convertedData;
        }
Пример #5
0
        private void _OnFileChanged(object sender, FilePropertyChangedEventArgs e)
        {
            Common.FileInfo source = sender as Common.FileInfo;
            if (source == null)
            {
                return;
            }

            try
            {
                //update the file record in the database
                List <DBPredicate> updateValues = new List <DBPredicate>();
                updateValues.Add(new DBPredicate("access_count", DBOperator.EQUALS, source.AccessCount.ToString()));
                updateValues.Add(new DBPredicate("favorite", DBOperator.EQUALS, source.IsFavorite ? "1" : "0"));
                updateValues.Add(new DBPredicate("hidden", DBOperator.EQUALS, source.IsHidden ? "1" : "0"));
                m_database.Update("FileCache", updateValues, new DBPredicate("id", DBOperator.EQUALS, source.ID.ToString()));
            }
            catch (Exception crap)
            {
                Settings.SessionLog += crap.Message + "\n";
                Debug.WriteLine(crap.Message);
            }

            if (LatestSearchResult == null)
            {
                return;
            }

            /*re-calculate the weight of the file and apply the changes by re-sorting the search result list*/
            double initialWeight = source.Weight;

            source.CalculateWeight(LatestSearchResult.SearchTerm);
            if (initialWeight != source.Weight && Settings.SortImmediatelyOnFileChange && m_limitedResultCollection.Count > 1)
            {
                try
                {
                    //if the first element was promoted, or the last element demoted no sorting is neccessary
                    if ((initialWeight < source.Weight && m_limitedResultCollection[0].SearchResultContent == source) ||
                        (initialWeight > source.Weight && m_limitedResultCollection[m_limitedResultCollection.Count - 1].SearchResultContent == source))
                    {
                        return;
                    }

                    int newIndex = -1;
                    int oldIndex = -1;
                    for (int i = 0; i < m_limitedResultCollection.Count; i++)
                    {
                        if (m_limitedResultCollection[i].SearchResultContent == source)
                        {
                            oldIndex = i;
                            if (newIndex != -1)
                            {
                                break;
                            }
                        }
                        else if (newIndex == -1)
                        {
                            if ((m_limitedResultCollection[i].SearchResultContent.Weight <= source.Weight))
                            {
                                newIndex = i;
                                if (oldIndex != -1)
                                {
                                    break;
                                }
                            }
                        }
                    }

                    if (oldIndex == -1)
                    {
                        return;
                    }

                    if (newIndex == -1)
                    {
                        newIndex = m_limitedResultCollection.Count - 1;
                    }

                    //handle case where first item was demoted but is still the largest element (normally it would be swapped with the second element)
                    if ((oldIndex != 0 || newIndex != 1) || (m_limitedResultCollection[newIndex].SearchResultContent.Weight >= m_limitedResultCollection[oldIndex].SearchResultContent.Weight))
                    {
                        m_limitedResultCollection.Move(oldIndex, newIndex);
                    }
                }
                catch (Exception ex)
                {
                    Debug.WriteLine(ex.Message);
                }
            }
        }