Beispiel #1
0
 /// <summary>
 /// get dataset for single item reindex via event
 /// </summary>
 /// <param name="filePath"></param>
 /// <param name="docIndex"></param>
 /// <returns></returns>
 public static Dictionary<string, string> GetDataSet(string filePath, int docIndex)
 {
     var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };
     var fileInfo = new FileInfo(filePath);
     simpleDataSet = MapFileToSimpleDataIndexItem(fileInfo, simpleDataSet, docIndex, "documentation");
     return simpleDataSet.RowData;
 }
Beispiel #2
0
        public static SimpleDataSet MapFileToSimpleDataIndexItem(FileInfo file, SimpleDataSet simpleDataSet, int index, string indexType)
        {
            var lines = new List<string>();
            lines.AddRange(File.ReadAllLines(file.FullName));

            var body = lines.Any()
                ? umbraco.library.StripHtml(RemoveSpecialCharacters(string.Join("", lines)))
                : string.Empty;

            var firstHeadline = lines.FirstOrDefault(x => x.StartsWith("#"));
            var headLine = firstHeadline ?? file.FullName.Substring(file.FullName.LastIndexOf("\\", StringComparison.Ordinal) + 1).Replace(".md", string.Empty).Replace("-", " ");

            simpleDataSet.NodeDefinition.NodeId = index;
            simpleDataSet.NodeDefinition.Type = indexType;

            simpleDataSet.RowData.Add("body", body);
            simpleDataSet.RowData.Add("nodeName", RemoveSpecialCharacters(headLine));
            simpleDataSet.RowData.Add("updateDate", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("nodeTypeAlias", "documentation");

            simpleDataSet.RowData.Add("dateCreated", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));

            //TODO: This will always be exactly the same since all files are written at the same time IIRC
            simpleDataSet.RowData.Add("Path", file.FullName);
            simpleDataSet.RowData.Add("searchAblePath", file.FullName.Replace("\\", " ").Replace(":", ""));
            simpleDataSet.RowData.Add("url", BuildUrl(file.FullName));
            return simpleDataSet;
        }
Beispiel #3
0
        public static SimpleDataSet MapFileToSimpleDataIndexItem(FileInfo file, SimpleDataSet simpleDataSet, int index, string indexType)
        {
            var lines = new List<string>();
            lines.AddRange(File.ReadAllLines(file.FullName));
            var headLine = string.Empty;
            var body = string.Empty;
            if (lines.Count > 0)
            {
                headLine = RemoveSpecialCharacters(lines[0]);
                lines.RemoveAt(0);
                body = umbraco.library.StripHtml( RemoveSpecialCharacters(string.Join("", lines)) );
            }

            simpleDataSet.NodeDefinition.NodeId = index;
            simpleDataSet.NodeDefinition.Type = indexType;

            simpleDataSet.RowData.Add("body", body);
            simpleDataSet.RowData.Add("nodeName", headLine);
            simpleDataSet.RowData.Add("updateDate", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("nodeTypeAlias", "documentation");

            simpleDataSet.RowData.Add("dateCreated", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));

            //TODO: This will always be exactly the same since all files are written at the same time IIRC
            simpleDataSet.RowData.Add("Path", file.FullName);
            simpleDataSet.RowData.Add("searchAblePath", file.FullName.Replace("\\", " ").Replace(":", ""));
            simpleDataSet.RowData.Add("url", BuildUrl(file.FullName));
            return simpleDataSet;
        }
        public IEnumerable<SimpleDataSet> GetAllData(string indexType)
        {
            var config = FileIndexerConfig.Settings;
            var fullPath = HttpContext.Current.Server.MapPath(config.DirectoryToIndex);

            var directory = new DirectoryInfo(fullPath);

            var files = config.Recursive ? directory.GetFiles(config.SupportedFileTypes, SearchOption.AllDirectories) : directory.GetFiles(config.SupportedFileTypes);

            var dataSets = new List<SimpleDataSet>();
            var i = 1; //unique id for each doc

            foreach (var file in files)
            {
                try
                {
                    var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };

                    simpleDataSet = ExamineHelper.MapFileToSimpleDataIndexItem(file, simpleDataSet, i, indexType);

                    dataSets.Add(simpleDataSet);
                }
                catch (Exception ex)
                {
                    Log.Add(LogTypes.Error, i, "error processing file  " + file.FullName + " " + ex);
                }

                i++;
            }

            return dataSets;
        }
Beispiel #5
0
        public static SimpleDataSet MapFileToSimpleDataIndexItem(FileInfo file, SimpleDataSet simpleDataSet, int index, string indexType)
        {
            var lines = new List<string>();
            lines.AddRange(File.ReadAllLines(file.FullName));
            var headLine = string.Empty;
            var body = string.Empty;
            if (lines.Count > 0)
            {
                headLine = RemoveSpecialCharacters(lines[0]);
                lines.RemoveAt(0);
                body = RemoveSpecialCharacters(string.Join("", lines));
            }

            simpleDataSet.NodeDefinition.NodeId = index;
            simpleDataSet.NodeDefinition.Type = indexType;
            simpleDataSet.RowData.Add("Body", body);
            simpleDataSet.RowData.Add("Title", headLine);
            simpleDataSet.RowData.Add("dateCreated", file.CreationTime.ToString("yyyy-MM-dd-HH:mm:ss"));
            simpleDataSet.RowData.Add("dateCreatedSearchAble", file.CreationTime.SerializeForLucene());
            simpleDataSet.RowData.Add("Path", file.FullName);
            simpleDataSet.RowData.Add("searchAblePath", file.FullName.Replace("\\", " ").Replace(":", ""));
            simpleDataSet.RowData.Add("nodeTypeAlias", "document");
            simpleDataSet.RowData.Add("url", BuildUrl(file.FullName));

            return simpleDataSet;
        }
        public SimpleDataSet CreateNewDocument(int id)
        {
            var ts = new TopicService(ApplicationContext.Current.DatabaseContext);

            var forumTopic = ts.QueryById(id);
            var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };
            return MapTopicToSimpleDataIndexItem(forumTopic, simpleDataSet, forumTopic.Id, "forum");
        }
        public SimpleDataSet MapProjectToSimpleDataIndexItem(IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
            int karma, IEnumerable<WikiFile> files, int downloads, IEnumerable<string> compatVersions)
        {
            simpleDataSet.NodeDefinition.NodeId = project.Id;
            simpleDataSet.NodeDefinition.Type = indexType;

            var desciption = project.GetPropertyValue<string>("description");
            if (!string.IsNullOrEmpty(desciption))
            {
                simpleDataSet.RowData.Add("body", umbraco.library.StripHtml(desciption));
            }
            simpleDataSet.RowData.Add("nodeName", project.Name);
            simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("nodeTypeAlias", "project");
            simpleDataSet.RowData.Add("url", project.Url );
            simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue<string>("packageGuid"));
            simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue<string>("worksOnUaaS"));

            var imageFile = string.Empty;
            if (project.HasValue("defaultScreenshotPath"))
            {
                imageFile = project.GetPropertyValue<string>("defaultScreenshotPath");
            }
            if(string.IsNullOrWhiteSpace(imageFile))
            {
                var image = files.FirstOrDefault(x => x.FileType == "screenshot");
                if (image != null)
                    imageFile = image.Path;
            }
            //Clean up version data before its included in the index
            int o;
            var version = project.GetProperty("compatibleVersions").Value;
            var versions = version.ToString().ToLower()
                            .Replace("nan", "")
                            .Replace("saved", "")
                            .Replace("v", "")
                            .Trim(',').Split(',')
                            .Where(x => int.TryParse(x, out o))
                            .Select(x => (decimal.Parse(x.PadRight(3, '0') ) / 100));

            //popularity for sorting number = downloads + karma * 100;
            var pop = downloads + (karma * 100);

            simpleDataSet.RowData.Add("popularity", pop.ToString());
            simpleDataSet.RowData.Add("karma", karma.ToString());
            simpleDataSet.RowData.Add("downloads", downloads.ToString());
            simpleDataSet.RowData.Add("image", imageFile);

            //now we need to add the versions and compat versions
            // first, this is the versions that the project has files tagged against
            simpleDataSet.RowData.Add("versions", string.Join(",", versions));

            //then we index the versions that the project has actually been flagged as compatible against
            simpleDataSet.RowData.Add("compatVersions", string.Join(",", compatVersions));

            return simpleDataSet;
        }
        public IEnumerable<SimpleDataSet> GetAllData(string indexType)
        { 
            var ts = new TopicService(ApplicationContext.Current.DatabaseContext);

            foreach (var topic in ts.QueryAll(maxCount:int.MaxValue))
            {
                //Add the item to the index..
                var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };
                yield return MapTopicToSimpleDataIndexItem(topic, simpleDataSet, topic.Id, "forum");
            }
        }
        public static SimpleDataSet MapTopicToSimpleDataIndexItem(ReadOnlyTopic topic, SimpleDataSet simpleDataSet, int id, string indexType)
        {
            //First generate the accumulated comment text:
            var commentText = string.Empty;
            foreach (var currentComment in topic.Comments.Where(c => c.IsSpam == false))
                commentText += currentComment.Body;

            var body = library.StripHtml(topic.Body + commentText);

            simpleDataSet.NodeDefinition.NodeId = id;
            simpleDataSet.NodeDefinition.Type = indexType;

            simpleDataSet.RowData.Add("body", body);
            if (!string.IsNullOrEmpty(commentText))
            {
                simpleDataSet.RowData.Add("comments", commentText);    
            }
            simpleDataSet.RowData.Add("nodeName", topic.Title);
            simpleDataSet.RowData.Add("updateDate", topic.Updated.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("nodeTypeAlias", "forum");
            
            simpleDataSet.RowData.Add("urlName", topic.UrlName);

            simpleDataSet.RowData.Add("createDate", topic.Created.ToString("yyyy-MM-dd HH:mm:ss"));

            simpleDataSet.RowData.Add("latestCommentId", topic.LatestComment.ToString());
            simpleDataSet.RowData.Add("latestReplyAuthorId", topic.LatestReplyAuthor.ToString());
            if (!string.IsNullOrEmpty(topic.LastReplyAuthorName))
            {
                simpleDataSet.RowData.Add("latestReplyAuthorName", topic.LastReplyAuthorName);    
            }
            

            simpleDataSet.RowData.Add("authorId", topic.MemberId.ToString());
            if (!string.IsNullOrEmpty(topic.AuthorName))
            {
                simpleDataSet.RowData.Add("authorName", topic.AuthorName);    
            }
            
            simpleDataSet.RowData.Add("parentId", topic.ParentId.ToString());
            simpleDataSet.RowData.Add("replies", topic.Replies.ToString());

            simpleDataSet.RowData.Add("locked", topic.Locked.ToString());
            simpleDataSet.RowData.Add("solved", topic.Answer.ToString());

            simpleDataSet.RowData.Add("version", topic.Version.ToString());

            return simpleDataSet;
        }
Beispiel #10
0
        private void UpdateProjectExamineIndex(IPublishedContent content, int downloads)
        {
            var simpleDataSet = new SimpleDataSet
            {
                NodeDefinition = new IndexedNode(),
                RowData = new Dictionary<string, string>()
            };

            var karma = Utils.GetProjectTotalVotes(content.Id);
            var files = WikiFile.CurrentFiles(content.Id);
            var compatVersions = Utils.GetProjectCompatibleVersions(content.Id);

            var simpleDataIndexer = (SimpleDataIndexer)ExamineManager.Instance.IndexProviderCollection["projectIndexer"];
            simpleDataSet = ((ProjectNodeIndexDataService)simpleDataIndexer.DataService)
                .MapProjectToSimpleDataIndexItem(content, simpleDataSet, "project", karma, files, downloads, compatVersions);

            var xml = simpleDataSet.RowData.ToExamineXml(simpleDataSet.NodeDefinition.NodeId, simpleDataSet.NodeDefinition.Type);
            simpleDataIndexer.ReIndexNode(xml, "project");
        }
        public IEnumerable<SimpleDataSet> GetAllData(string indexType)
        {
            //Before getting all data, we need to make sure that the docs are available from GitHub
            ZipDownloader.EnsureGitHubDocs();

            var config = DocumentationIndexConfig.Settings;
            var fullPath = HttpContext.Current.Server.MapPath(config.DirectoryToIndex);

            var directory = new DirectoryInfo(fullPath);

            var files = config.Recursive ? directory.GetFiles(config.SupportedFileTypes, SearchOption.AllDirectories) : directory.GetFiles(config.SupportedFileTypes);

            var i = 0; //unique id for each doc

            foreach (var file in files)
            {
                i++;
                var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };
                simpleDataSet = ExamineHelper.MapFileToSimpleDataIndexItem(file, simpleDataSet, i, indexType);
                yield return simpleDataSet;
            }
        }
        public IEnumerable<SimpleDataSet> GetAllData(string indexType)
        {   
            var data = new List<SimpleDataSet>();
            var count = 1;

            LogHelper.Info<BookshelfExamineDataService>("Building index...");

            foreach (var bookPath in _getBooksAsDirectories())
            {
                var files = bookPath.GetFilesRecursively(Constants.ALLOWED_FILE_EXTENSIONS);

                foreach (var file in files)
                {
                    var dataset = new SimpleDataSet()
                    {
                        NodeDefinition = new IndexedNode()
                        {
                            Type = "Bookshelf",
                            NodeId = count
                        }
                    };

                    dataset.RowData = new Dictionary<string, string>()
                    {
                        {"book", Path.GetFileName(bookPath)},
                        {"path", bookPath.ToWebPath()},
                        {"title", Path.GetFileNameWithoutExtension(file)},
                        {"text", File.ReadAllText(file)},
                        {"url", "/umbraco/#/UmbracoBookshelf/UmbracoBookshelfTree/file/" + file.ToWebPath().Replace("%2F", "%252F").Replace("%20F", "%2520F")} //total hack job here b/c of some sort of double encoding somewhere
                    };

                    data.Add(dataset);
                    count++;
                }
            }

            return data;
        }
        public SimpleDataSet MapProjectToSimpleDataIndexItem(IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
            int projectVotes, WikiFile[] files, int downloads, IEnumerable<string> compatVersions)
        {
            var isLive = project.GetPropertyValue<bool>("projectLive");
            var isApproved = project.GetPropertyValue<bool>("approved");

            var minimumVersionStrict = string.Empty;
            var currentFileId = project.GetPropertyValue<int>("file");
            if (currentFileId > 0)
            {
                var currentFile = files.FirstOrDefault(x => x.Id == currentFileId);
                if (currentFile != null)
                    minimumVersionStrict = currentFile.MinimumVersionStrict;
            }

            simpleDataSet.NodeDefinition.NodeId = project.Id;
            simpleDataSet.NodeDefinition.Type = indexType;

            simpleDataSet.RowData.Add("body", project.GetPropertyValue<string>("description"));
            simpleDataSet.RowData.Add("nodeName", project.Name);
            simpleDataSet.RowData.Add("categoryFolder", project.Parent.Name.ToLowerInvariant().Trim());
            simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
            simpleDataSet.RowData.Add("nodeTypeAlias", "project");
            simpleDataSet.RowData.Add("url", project.Url);
            simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue<string>("packageGuid"));
            simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue<string>("worksOnUaaS"));
            simpleDataSet.RowData.Add("minimumVersionStrict", minimumVersionStrict);

            var imageFile = string.Empty;
            if (project.HasValue("defaultScreenshotPath"))
            {
                imageFile = project.GetPropertyValue<string>("defaultScreenshotPath");
            }
            if (string.IsNullOrWhiteSpace(imageFile))
            {
                var image = files.FirstOrDefault(x => x.FileType == "screenshot");
                if (image != null)
                    imageFile = image.Path;
            }

            //Clean up version data before its included in the index, the reason we have to do this
            // is due to the way the version data is stored, you can see it in uVersion.config - it's super strange
            // because of the 3 digit nature but when it doesn't end with a '0' it's actually just the major/minor version
            // so we have to do all of this parsing.
            var version = project.GetPropertyValue<string>("compatibleVersions") ?? string.Empty;
            var cleanedVersions = version.ToLower()
                .Replace("nan", "")
                .Replace("saved", "")
                .Replace("v", "")
                .Trim(',')
                .Split(',')
                //it's stored as an int like 721 (for version 7.2.1)
                .Where(x => x.Length <= 3 && x.Length > 0)
                //pad it out to 3 digits
                .Select(x => x.PadRight(3, '0'))
                .Select(x =>
                {
                    int o;
                    if (int.TryParse(x, out o))
                    {
                        //if it ends with '0', that means it's a X.X.X version
                        // if it does not end with '0', that means that the last 2 digits are the
                        // Minor part of the version
                        return x.EndsWith("0")
                            ? string.Format("{0}.{1}.{2}", x[0], x[1], 0)
                            : string.Format("{0}.{1}.{2}", x[0], x.Substring(1), 0);
                    }
                    return null;
                })
                .Where(x => x != null);

            var cleanedCompatVersions = compatVersions.Select(x => x.Replace("nan", "")
                .Replace("saved", "")
                .Replace("nan", "")
                .Replace("v", "")
                .Replace(".x", "")
                .Trim(','));

            //popularity for sorting number = downloads + karma * 100;
            //TODO: Change score so that we take into account:
            // - recently updated
            // - works on latest umbraco versions
            // - works on uaas
            // - has a forum
            // - has source code link
            // - open for collab / has collaborators
            // - download count in a recent timeframe - since old downloads should count for less

            var pop = downloads + (projectVotes * 100);

            simpleDataSet.RowData.Add("popularity", pop.ToString());
            simpleDataSet.RowData.Add("karma", projectVotes.ToString());
            simpleDataSet.RowData.Add("downloads", downloads.ToString());
            simpleDataSet.RowData.Add("image", imageFile);

            var packageFiles = files.Count(x => x.FileType == "package");
            simpleDataSet.RowData.Add("packageFiles", packageFiles.ToString());

            simpleDataSet.RowData.Add("projectLive", isLive ? "1" : "0");
            simpleDataSet.RowData.Add("approved", isApproved ? "1" : "0");

            //now we need to add the versions and compat versions
            // first, this is the versions that the project has files tagged against
            simpleDataSet.RowData.Add("versions", string.Join(",", cleanedVersions));
            //then we index the versions that the project has actually been flagged as compatible against
            simpleDataSet.RowData.Add("compatVersions", string.Join(",", cleanedCompatVersions));

            return simpleDataSet;
        }
        public IEnumerable<SimpleDataSet> GetAllData(string indexType)
        {
            var umbContxt = EnsureUmbracoContext();

            var projects = umbContxt.ContentCache.GetByXPath("//Community/Projects//Project [projectLive='1']").ToArray();

            var allProjectIds = projects.Select(x => x.Id).ToArray();
            var allProjectKarma = Utils.GetProjectTotalVotes();
            var allProjectWikiFiles = WikiFile.CurrentFiles(allProjectIds);
            var allProjectDownloads = Utils.GetProjectTotalDownload();
            var allCompatVersions = Utils.GetProjectCompatibleVersions();

            foreach (var project in projects)
            {
                LogHelper.Debug(this.GetType(), "Indexing " + project.Name);

                var simpleDataSet = new SimpleDataSet { NodeDefinition = new IndexedNode(), RowData = new Dictionary<string, string>() };

                var projectDownloads = allProjectDownloads.ContainsKey(project.Id) ? allProjectDownloads[project.Id] : 0;
                var projectKarma = allProjectKarma.ContainsKey(project.Id) ? allProjectKarma[project.Id] : 0;
                var projectFiles = allProjectWikiFiles.ContainsKey(project.Id) ? allProjectWikiFiles[project.Id].ToArray() : new WikiFile[] { };
                var projectVersions = allCompatVersions.ContainsKey(project.Id) ? allCompatVersions[project.Id] : Enumerable.Empty<string>();

                yield return MapProjectToSimpleDataIndexItem(project, simpleDataSet, indexType, projectKarma, projectFiles, projectDownloads, projectVersions);
            }
        }