public SimpleDataSet <N> GenerateNewDataSet <N>(ConvertDataFunction <N> func)
{
SimpleDataSet <N> newSDS = new SimpleDataSet <N>( );
for (int i = 0; i < _dataList.Count; ++i)
{
N newData = func(_dataList[i].Data, i);
newSDS.AddDataPoint(_dataList[i].Time, newData);
}
return(newSDS);
}
internal static SimpleDataSet MapFileToSimpleDataIndexItem(FileInfo file, SimpleDataSet simpleDataSet, int index, string indexType)
{
var lines = new List <string>();
lines.AddRange(File.ReadAllLines(file.FullName));
// don't parse empty files
if (lines.Count == 0)
{
Umbraco.Core.Logging.LogHelper.Info(typeof(ExamineHelper), "Skipping file {0}", () => file.FullName);
return(simpleDataSet);
}
// remove first empty lines
while (string.IsNullOrWhiteSpace(lines.ElementAt(0)))
{
lines.RemoveAt(0);
}
int secondYamlMarker = AddYamlFields(simpleDataSet, lines);
// build body
var bodyLines = lines.GetRange(secondYamlMarker + 1, lines.Count - secondYamlMarker - 1);
var body = bodyLines.Any()
? RemoveSpecialCharacters(string.Join(" ", bodyLines)).StripHtml()
: string.Empty;
// extract the first headline
var firstHeadline = lines.FirstOrDefault(x => x.StartsWith("#"));
var headLine = firstHeadline ?? file.FullName.Substring(file.FullName.LastIndexOf("\\", StringComparison.Ordinal) + 1).Replace(".md", string.Empty).Replace("-", " ");
simpleDataSet.NodeDefinition.NodeId = index;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("body", body);
simpleDataSet.RowData.Add("nodeName", RemoveSpecialCharacters(headLine));
simpleDataSet.RowData.Add("updateDate", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "documentation");
simpleDataSet.RowData.Add("dateCreated", file.CreationTime.ToString("yyyy-MM-dd HH:mm:ss"));
//TODO: This will always be exactly the same since results all files are written at the same time IIRC
simpleDataSet.RowData.Add("Path", file.FullName);
simpleDataSet.RowData.Add("searchAblePath", file.FullName.Replace("\\", " ").Replace(":", ""));
simpleDataSet.RowData.Add("url", BuildUrl(file.FullName));
// Used to exclude broken link page from the search results.
if (file.Name == "broken-link.md")
{
simpleDataSet.RowData.Clear();
}
return(simpleDataSet);
}
private void UpdateProjectExamineIndex(IPublishedContent content, int downloads)
{
if (content == null)
{
return;
}
var simpleDataSet = new SimpleDataSet
{
NodeDefinition = new IndexedNode(),
RowData = new Dictionary <string, string>()
};
var projectVotes = Utils.GetProjectTotalVotes(content.Id);
var files = WikiFile.CurrentFiles(content.Id).ToArray();
var compatVersions = Utils.GetProjectCompatibleVersions(content.Id) ?? new List <string>();
var downloadStats = WikiFile.GetMonthlyDownloadStatsByProject(
content.Id,
DateTime.Now.Subtract(TimeSpan.FromDays(365)));
var nugetService = new OurUmbraco.Community.Nuget.NugetPackageDownloadService();
var nugetPackageId = nugetService.GetNuGetPackageId(content);
int?dailyNugetDownLoads = null;
if (!nugetPackageId.IsNullOrWhiteSpace())
{
var nugetDownloads = nugetService.GetNugetPackageDownloads();
var packageInfo = nugetDownloads.FirstOrDefault(x => x.PackageId == nugetPackageId);
if (packageInfo != null)
{
downloads += packageInfo.TotalDownLoads;
dailyNugetDownLoads = packageInfo.AverageDownloadPerDay;
}
}
var simpleDataIndexer = (SimpleDataIndexer)ExamineManager.Instance.IndexProviderCollection["projectIndexer"];
simpleDataSet = ((ProjectNodeIndexDataService)simpleDataIndexer.DataService)
.MapProjectToSimpleDataIndexItem(downloadStats, DateTime.Now, content, simpleDataSet, "project", projectVotes, files, downloads, compatVersions, dailyNugetDownLoads);
if (simpleDataSet.NodeDefinition.Type == null)
{
simpleDataSet.NodeDefinition.Type = "project";
}
var xml = simpleDataSet.RowData.ToExamineXml(simpleDataSet.NodeDefinition.NodeId, simpleDataSet.NodeDefinition.Type);
simpleDataIndexer.ReIndexNode(xml, "project");
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var umbContxt = EnsureUmbracoContext();
var projects = umbContxt.ContentCache.GetByXPath("//Community/Projects//Project [projectLive='1']").ToArray();
var nugetService = new NugetPackageDownloadService();
var nugetDownloads = nugetService.GetNugetPackageDownloads();
var allProjectIds = projects.Select(x => x.Id).ToArray();
var allProjectKarma = Utils.GetProjectTotalVotes();
var allProjectWikiFiles = WikiFile.CurrentFiles(allProjectIds);
var allProjectDownloads = Utils.GetProjectTotalPackageDownload();
var allCompatVersions = Utils.GetProjectCompatibleVersions();
var mostRecentDownloadDate = WikiFile.GetMostRecentDownloadDate();
var downloadStats = WikiFile.GetMonthlyDownloadStatsByProject(mostRecentDownloadDate.Subtract(TimeSpan.FromDays(365)));
foreach (var project in projects)
{
LogHelper.Debug(this.GetType(), "Indexing " + project.Name);
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
var projectDownloads = allProjectDownloads.ContainsKey(project.Id) ? allProjectDownloads[project.Id] : 0;
var projectKarma = allProjectKarma.ContainsKey(project.Id) ? allProjectKarma[project.Id] : 0;
var projectFiles = allProjectWikiFiles.ContainsKey(project.Id) ? allProjectWikiFiles[project.Id].ToArray() : new WikiFile[] { };
var projectVersions = allCompatVersions.ContainsKey(project.Id) ? allCompatVersions[project.Id] : Enumerable.Empty <string>();
var nugetPackageId = nugetService.GetNuGetPackageId(project);
int?dailyNugetDownLoads = null;
if (!string.IsNullOrWhiteSpace(nugetPackageId))
{
var packageInfo = nugetDownloads.FirstOrDefault(x => x.PackageId == nugetPackageId);
if (packageInfo != null)
{
projectDownloads += packageInfo.TotalDownLoads;
dailyNugetDownLoads = packageInfo.AverageDownloadPerDay;
}
}
yield return(MapProjectToSimpleDataIndexItem(
downloadStats,
mostRecentDownloadDate,
project, simpleDataSet, indexType, projectKarma, projectFiles, projectDownloads, projectVersions, dailyNugetDownLoads));
}
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var ts = new TopicService(ApplicationContext.Current.DatabaseContext);
foreach (var topic in ts.QueryAll(maxCount:int.MaxValue))
{
//Add the item to the index..
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
yield return(MapTopicToSimpleDataIndexItem(topic, simpleDataSet, topic.Id, "forum"));
}
}
private SimpleDataSet CreateIndexItemFromJob(int fakeNodeId, Job job, string indexType)
{
LogHelper.Info <BaseJobsIndexer>($"Building Examine index item for job '{job.Id}'");
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
simpleDataSet.NodeDefinition.NodeId = fakeNodeId;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("id", job.Id);
simpleDataSet.RowData.Add("reference", job.Reference);
simpleDataSet.RowData.Add("title", _stopWordsRemover.Filter(job.JobTitle));
simpleDataSet.RowData.Add("titleDisplay", job.JobTitle);
simpleDataSet.RowData.Add("organisation", _stopWordsRemover.Filter(job.Organisation));
simpleDataSet.RowData.Add("organisationDisplay", job.Organisation);
simpleDataSet.RowData.Add("location", _stopWordsRemover.Filter(job.Location));
simpleDataSet.RowData.Add("locationDisplay", job.Location); // because Somewhere-on-Sea needs to lose the "on" for searching but keep it for display
simpleDataSet.RowData.Add("salary", _tagSanitiser.StripTags(_stopWordsRemover.Filter(job.Salary.SalaryRange)));
simpleDataSet.RowData.Add("salaryRange", _stopWordsRemover.Filter(job.Salary.SearchRange));
simpleDataSet.RowData.Add("salaryMin", job.Salary.MinimumSalary?.ToString("D7") ?? String.Empty);
simpleDataSet.RowData.Add("salaryMax", job.Salary.MaximumSalary?.ToString("D7") ?? String.Empty);
simpleDataSet.RowData.Add("salarySort", (job.Salary.MinimumSalary?.ToString("D7") ?? String.Empty) + " " + (job.Salary.MaximumSalary?.ToString("D7") ?? String.Empty) + " " + _stopWordsRemover.Filter(job.Salary.SalaryRange));
simpleDataSet.RowData.Add("closingDate", job.ClosingDate.Value.ToIso8601DateTime());
simpleDataSet.RowData.Add("closingDateDisplay", job.ClosingDate.Value.ToIso8601DateTime());
simpleDataSet.RowData.Add("jobType", _stopWordsRemover.Filter(job.JobType));
simpleDataSet.RowData.Add("jobTypeDisplay", job.JobType);
simpleDataSet.RowData.Add("contractType", _stopWordsRemover.Filter(job.ContractType));
simpleDataSet.RowData.Add("department", _stopWordsRemover.Filter(job.Department));
simpleDataSet.RowData.Add("departmentDisplay", job.Department);
simpleDataSet.RowData.Add("fullTime", job.WorkPattern.IsFullTime.ToString());
simpleDataSet.RowData.Add("partTime", job.WorkPattern.IsPartTime.ToString());
simpleDataSet.RowData.Add("workPattern", job.WorkPattern.ToString());
if (job.AdvertHtml != null)
{
simpleDataSet.RowData.Add("fullText", _tagSanitiser.StripTags(job.AdvertHtml.ToHtmlString()));
simpleDataSet.RowData.Add("fullHtml", job.AdvertHtml.ToHtmlString());
}
if (job.AdditionalInformationHtml != null)
{
simpleDataSet.RowData.Add("additionalInfo", job.AdditionalInformationHtml.ToHtmlString());
}
if (job.EqualOpportunitiesHtml != null)
{
simpleDataSet.RowData.Add("equalOpportunities", job.EqualOpportunitiesHtml.ToHtmlString());
}
simpleDataSet.RowData.Add("applyUrl", job.ApplyUrl?.ToString());
return(simpleDataSet);
}
private static int AddYamlFields(SimpleDataSet simpleDataSet, List <string> lines)
{
// Check if the first line is a YAML marker
// YAML is only accepted if it's on the top of the document
// because empty lines are already removed, the index needs to be 0
bool hasYaml = lines.ElementAt(0) == "---";
int secondYamlMarker = 0;
if (hasYaml)
{
// Find the "next" triple dash starting from the second line
secondYamlMarker = lines.IndexOf("---", 1);
// add all yaml together and parse YAML meta data
YamlMetaData yamlMetaData = new YamlMetaData();
if (secondYamlMarker > 0)
{
// we found a second marker, so we have YAML data available
var yamlInput = new StringBuilder();
for (int i = 1; i < secondYamlMarker; i++)
{
yamlInput.AppendLine(lines.ElementAt(i));
}
;
// Try to convert the YAML text to a strongly typed model using YamlDotNet
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention())
.IgnoreUnmatchedProperties()
.Build();
yamlMetaData = deserializer.Deserialize <YamlMetaData>(yamlInput.ToString());
}
// Add Yaml stuff to the LUCENE index
simpleDataSet.RowData.Add("tags", yamlMetaData.Tags);
simpleDataSet.RowData.Add("keywords", yamlMetaData.Keywords);
simpleDataSet.RowData.Add("versionFrom", yamlMetaData.VersionFrom);
simpleDataSet.RowData.Add("versionTo", yamlMetaData.VersionTo);
var matchingMajorVersions = CalculateMajorVersions(yamlMetaData);
simpleDataSet.RowData.Add("majorVersion", string.Join(" ", matchingMajorVersions));
}
else
{
// no YAML information, add the current version as majorVersion
simpleDataSet.RowData.Add("majorVersion", GetCurrentDocVersion().ToString());
}
return(secondYamlMarker);
}
public float QueryValue(float time)
{
SimpleDataSet <float> .SimpleDataEntry before = _data.GetBefore(time);
SimpleDataSet <float> .SimpleDataEntry after = _data.GetAfter(time);
if ((before != null) && (after != null))
{
float t = Mathf.InverseLerp(before.Time, after.Time, time);
float result = Mathf.Lerp(before.Data, after.Data, t);
return(result);
}
return(float.PositiveInfinity);
}
private static async Task <SimpleDataSet> CreateDataSetFromLookup(int fakeNodeId, string indexType, string group, JobSearchQuery query, JobsLookupValue lookupValue, IJobsDataProvider jobsDataProvider)
{
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
simpleDataSet.NodeDefinition.NodeId = fakeNodeId;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("id", lookupValue.Id);
simpleDataSet.RowData.Add("group", group);
simpleDataSet.RowData.Add("text", lookupValue.Text);
if (jobsDataProvider != null)
{
var jobs = await jobsDataProvider.ReadJobs(query);
simpleDataSet.RowData.Add("count", jobs.Count.ToString(CultureInfo.CurrentCulture));
}
return(simpleDataSet);
}
internal static SimpleDataSet Get <T>(T itemToIndex) where T : class, IExamineDocument
{
SimpleDataSet simpleDataSet = null;
var rowData = GetRowData(itemToIndex);
if (rowData != null)
{
simpleDataSet = new SimpleDataSet
{
NodeDefinition = new IndexedNode
{
NodeId = itemToIndex.Id,
Type = Constants.IndexType
},
RowData = rowData
};
}
return(simpleDataSet);
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var config = DocumentationIndexConfig.Settings;
var fullPath = HostingEnvironment.MapPath(config.DirectoryToIndex);
var directory = new DirectoryInfo(fullPath);
var files = config.Recursive ? directory.GetFiles(config.SupportedFileTypes, SearchOption.AllDirectories) : directory.GetFiles(config.SupportedFileTypes);
var i = 0; //unique id for each doc
foreach (var file in files)
{
i++;
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
simpleDataSet = ExamineHelper.MapFileToSimpleDataIndexItem(file, simpleDataSet, i, indexType);
yield return(simpleDataSet);
}
}
private void UpdateProjectExamineIndex(IPublishedContent content, int downloads)
{
var simpleDataSet = new SimpleDataSet
{
NodeDefinition = new IndexedNode(),
RowData = new Dictionary <string, string>()
};
var projectVotes = Utils.GetProjectTotalVotes(content.Id);
var files = WikiFile.CurrentFiles(content.Id).ToArray();
var compatVersions = Utils.GetProjectCompatibleVersions(content.Id);
var simpleDataIndexer = (SimpleDataIndexer)ExamineManager.Instance.IndexProviderCollection["projectIndexer"];
simpleDataSet = ((ProjectNodeIndexDataService)simpleDataIndexer.DataService)
.MapProjectToSimpleDataIndexItem(content, simpleDataSet, "project", projectVotes, files, downloads, compatVersions);
var xml = simpleDataSet.RowData.ToExamineXml(simpleDataSet.NodeDefinition.NodeId, simpleDataSet.NodeDefinition.Type);
simpleDataIndexer.ReIndexNode(xml, "project");
}
private static async Task <IEnumerable <SimpleDataSet> > CreateDataSetFromLookupValues(string indexType, string groupKey, int fakeNodeId, IList <JobsLookupValue> lookupValues, Action <JobsLookupValue, JobSearchQuery> addLookupToQuery, IJobsDataProvider jobsDataProvider)
{
var dataSets = new List <SimpleDataSet>();
if (lookupValues != null)
{
foreach (var lookupValue in lookupValues)
{
JobSearchQuery query = null;
if (addLookupToQuery != null)
{
query = new JobSearchQuery();
addLookupToQuery(lookupValue, query);
query.ClosingDateFrom = DateTime.Today;
}
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
simpleDataSet.NodeDefinition.NodeId = fakeNodeId;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("id", lookupValue.LookupValueId);
simpleDataSet.RowData.Add("group", groupKey);
simpleDataSet.RowData.Add("text", lookupValue.Text);
if (jobsDataProvider != null)
{
var jobs = await jobsDataProvider.ReadJobs(query).ConfigureAwait(false);
simpleDataSet.RowData.Add("count", jobs.TotalJobs.ToString(CultureInfo.CurrentCulture));
}
dataSets.Add(simpleDataSet);
fakeNodeId++;
}
}
return(dataSets);
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
using (var umbracoDb = new Database("umbracoDbDSN"))
{
var towns = umbracoDb.Query <Town>("select * from uk_towns");
foreach (var town in towns)
{
var sds = new SimpleDataSet {
NodeDefinition = new IndexedNode()
};
sds.NodeDefinition.NodeId = town.Id;
sds.NodeDefinition.Type = indexType;
sds.RowData = town.ToSimpleDataSet().RowData;
yield return(sds);
}
}
}
private SimpleDataSet ToSimpleDataSet(SearchResult searchResult)
{
var simpleDataSet = new SimpleDataSet
{
RowData = new Dictionary <string, string>(),
NodeDefinition = new IndexedNode {
NodeId = searchResult.Id, Type = IndexType
}
};
simpleDataSet.RowData.Add("repository", searchResult["repository"]);
simpleDataSet.RowData.Add("state", searchResult["state"]);
simpleDataSet.RowData.Add("title", searchResult["title"]);
simpleDataSet.RowData.Add("createdAt", searchResult["createdAt"]);
simpleDataSet.RowData.Add("updatedAt", searchResult["updatedAt"]);
simpleDataSet.RowData.Add("closedAt", searchResult["closedAt"]);
simpleDataSet.RowData.Add("mergedAt", searchResult["mergedAt"]);
simpleDataSet.RowData.Add("userId", searchResult["userId"]);
simpleDataSet.RowData.Add("userLogin", searchResult["userLogin"]);
return(simpleDataSet);
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var config = DocumentationIndexConfig.Settings;
var fullPath = HostingEnvironment.MapPath(config.DirectoryToIndex);
var directory = new DirectoryInfo(fullPath);
var files = config.Recursive
? directory.GetFiles(config.SupportedFileTypes, SearchOption.AllDirectories)
: directory.GetFiles(config.SupportedFileTypes);
var i = 0; //unique id for each doc
foreach (var file in files)
{
i++;
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
try
{
simpleDataSet = ExamineHelper.MapFileToSimpleDataIndexItem(file, simpleDataSet, i, indexType);
}
catch (Exception ex)
{
Umbraco.Core.Logging.LogHelper.Error <DocumentationIndexDataService>(
$"Indexing docs - could not parse document {file.FullName}", ex);
if (System.Diagnostics.Debugger.IsAttached)
{
System.Diagnostics.Debugger.Break();
}
}
yield return(simpleDataSet);
}
Umbraco.Core.Logging.LogHelper.Info <DocumentationIndexDataService>(
$"Indexed documentation files: {0}", () => files.Length);
}
public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var data = new List <SimpleDataSet>();
var count = 1;
LogHelper.Info <BookshelfExamineDataService>("Building index...");
foreach (var bookPath in _getBooksAsDirectories())
{
var files = bookPath.GetFilesRecursively(Constants.ALLOWED_FILE_EXTENSIONS);
foreach (var file in files)
{
var dataset = new SimpleDataSet()
{
NodeDefinition = new IndexedNode()
{
Type = "Bookshelf",
NodeId = count
}
};
dataset.RowData = new Dictionary <string, string>()
{
{ "book", Path.GetFileName(bookPath) },
{ "path", bookPath.ToWebPath() },
{ "title", Path.GetFileNameWithoutExtension(file) },
{ "text", File.ReadAllText(file) },
{ "url", "/umbraco/#/UmbracoBookshelf/UmbracoBookshelfTree/file/" + file.ToWebPath().Replace("%2F", "%252F").Replace("%20F", "%2520F") } //total hack job here b/c of some sort of double encoding somewhere
};
data.Add(dataset);
count++;
}
}
return(data);
}
public SimpleDataSet MapProjectToSimpleDataIndexItem(
IDictionary <int, MonthlyProjectDownloads> projectDownloadStats,
DateTime mostRecentUpdateDate,
IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
int projectVotes, WikiFile[] files, int downloads, IEnumerable <string> compatVersions)
{
var isLive = project.GetPropertyValue <bool>("projectLive");
var isApproved = project.GetPropertyValue <bool>("approved");
var strictPackageFiles = PackageRepositoryService.GetAllStrictSupportedPackageVersions(files);
simpleDataSet.NodeDefinition.NodeId = project.Id;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("body", project.GetPropertyValue <string>("description"));
simpleDataSet.RowData.Add("nodeName", project.Name);
simpleDataSet.RowData.Add("categoryFolder", project.Parent.Name.ToLowerInvariant().Trim());
simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "project");
simpleDataSet.RowData.Add("url", project.Url);
simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue <string>("packageGuid"));
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue <string>("worksOnUaaS"));
var imageFile = string.Empty;
if (project.HasValue("defaultScreenshotPath"))
{
imageFile = project.GetPropertyValue <string>("defaultScreenshotPath");
}
if (string.IsNullOrWhiteSpace(imageFile))
{
var image = files.FirstOrDefault(x => x.FileType == "screenshot");
if (image != null)
{
imageFile = image.Path;
}
}
//Clean up version data before its included in the index, the reason we have to do this
// is due to the way the version data is stored, you can see it in uVersion.config - it's super strange
// because of the 3 digit nature but when it doesn't end with a '0' it's actually just the major/minor version
// so we have to do all of this parsing.
var version = project.GetPropertyValue <string>("compatibleVersions") ?? string.Empty;
var cleanedVersions = version.ToLower()
.Trim(',')
.Split(',')
.Select(x => x.GetFromUmbracoString(reduceToConfigured: false))
.Where(x => x != null);
var cleanedCompatVersions = compatVersions
.Select(x => x.GetFromUmbracoString(reduceToConfigured: false))
.Where(x => x != null);
var hasForum = project.Children.Any(x => x.IsVisible());
MonthlyProjectDownloads projStats = null;
projectDownloadStats.TryGetValue(project.Id, out projStats);
var points = new ProjectPopularityPoints(
mostRecentUpdateDate,
projStats,
project.CreateDate,
project.UpdateDate,
project.GetPropertyValue <bool>("worksOnUaaS"), hasForum,
project.GetPropertyValue <string>("sourceUrl").IsNullOrWhiteSpace() == false,
project.GetPropertyValue <bool>("openForCollab"),
downloads,
projectVotes);
var pop = points.Calculate();
simpleDataSet.RowData.Add("popularity", pop.ToString());
simpleDataSet.RowData.Add("karma", projectVotes.ToString());
simpleDataSet.RowData.Add("downloads", downloads.ToString());
simpleDataSet.RowData.Add("image", imageFile);
var packageFiles = files.Count(x => x.FileType == "package");
simpleDataSet.RowData.Add("packageFiles", packageFiles.ToString());
simpleDataSet.RowData.Add("projectLive", isLive ? "1" : "0");
simpleDataSet.RowData.Add("approved", isApproved ? "1" : "0");
//now we need to add the versions and compat versions
// first, this is the versions that the project has files tagged against
simpleDataSet.RowData.Add("versions", string.Join(",", cleanedVersions));
//then we index the versions that the project has actually been flagged as compatible against
simpleDataSet.RowData.Add("compatVersions", string.Join(",", cleanedCompatVersions));
simpleDataSet.RowData.Add("minimumVersionStrict", string.Join(",", strictPackageFiles.Select(x => x.MinUmbracoVersion.ToString(3))));
return(simpleDataSet);
}
public SimpleDataSet MapProjectToSimpleDataIndexItem(IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
int karma, IEnumerable <WikiFile> files, int downloads, IEnumerable <string> compatVersions)
{
simpleDataSet.NodeDefinition.NodeId = project.Id;
simpleDataSet.NodeDefinition.Type = indexType;
var desciption = project.GetPropertyValue <string>("description");
if (!string.IsNullOrEmpty(desciption))
{
simpleDataSet.RowData.Add("body", umbraco.library.StripHtml(desciption));
}
simpleDataSet.RowData.Add("nodeName", project.Name);
simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "project");
simpleDataSet.RowData.Add("url", project.Url);
simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue <string>("packageGuid"));
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue <string>("worksOnUaaS"));
var imageFile = string.Empty;
if (project.HasValue("defaultScreenshotPath"))
{
imageFile = project.GetPropertyValue <string>("defaultScreenshotPath");
}
if (string.IsNullOrWhiteSpace(imageFile))
{
var image = files.FirstOrDefault(x => x.FileType == "screenshot");
if (image != null)
{
imageFile = image.Path;
}
}
//Clean up version data before its included in the index
int o;
var version = project.GetProperty("compatibleVersions").Value;
var versions = version.ToString().ToLower()
.Replace("nan", "")
.Replace("saved", "")
.Replace("v", "")
.Trim(',').Split(',')
.Where(x => int.TryParse(x, out o))
.Select(x => (decimal.Parse(x.PadRight(3, '0')) / 100));
//popularity for sorting number = downloads + karma * 100;
var pop = downloads + (karma * 100);
simpleDataSet.RowData.Add("popularity", pop.ToString());
simpleDataSet.RowData.Add("karma", karma.ToString());
simpleDataSet.RowData.Add("downloads", downloads.ToString());
simpleDataSet.RowData.Add("image", imageFile);
//now we need to add the versions and compat versions
// first, this is the versions that the project has files tagged against
simpleDataSet.RowData.Add("versions", string.Join(",", versions));
//then we index the versions that the project has actually been flagged as compatible against
simpleDataSet.RowData.Add("compatVersions", string.Join(",", compatVersions));
return(simpleDataSet);
}
private SimpleDataSet CreateIndexItemFromJob(Job job, string indexType)
{
if (job.DatePublished > DateTime.UtcNow)
{
LogHelper.Info <BaseJobsIndexer>($"Ignoring job '{job.Id}' because it's publish date {job.DatePublished.ToIso8601DateTime()} is in the future.");
return(null);
}
LogHelper.Info <BaseJobsIndexer>($"Building Examine index item for job '{job.Id}'");
var salary = job.Salary.SalaryRange;
var salaryWithStopWords = salary;
if (!String.IsNullOrEmpty(salary))
{
if (TagSanitiser != null)
{
salary = TagSanitiser.StripTags(salary);
}
if (StopWordsRemover != null)
{
salary = StopWordsRemover.Filter(salary);
}
}
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
simpleDataSet.NodeDefinition.NodeId = job.Id;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("id", job.Id.ToString(CultureInfo.InvariantCulture));
simpleDataSet.RowData.Add("reference", job.Reference);
simpleDataSet.RowData.Add("numberOfPositions", job.NumberOfPositions?.ToString(CultureInfo.CurrentCulture));
simpleDataSet.RowData.Add("title", StopWordsRemover != null ? StopWordsRemover.Filter(job.JobTitle) : job.JobTitle);
simpleDataSet.RowData.Add("titleDisplay", job.JobTitle);
simpleDataSet.RowData.Add("logoUrl", job.LogoUrl?.ToString());
simpleDataSet.RowData.Add("organisation", StopWordsRemover != null ? StopWordsRemover.Filter(job.Organisation) : job.Organisation);
simpleDataSet.RowData.Add("organisationDisplay", job.Organisation);
simpleDataSet.RowData.Add("salary", salary);
simpleDataSet.RowData.Add("salaryDisplay", salaryWithStopWords); // so that it's not displayed with stop words removed
simpleDataSet.RowData.Add("salaryRange", StopWordsRemover != null ? StopWordsRemover.Filter(job.Salary.SearchRange) : job.Salary.SearchRange);
simpleDataSet.RowData.Add("salaryMin", job.Salary.MinimumSalary?.ToString("0000000.00").Replace(".", string.Empty) ?? String.Empty);
simpleDataSet.RowData.Add("salaryMax", job.Salary.MaximumSalary?.ToString("0000000.00").Replace(".", string.Empty) ?? String.Empty);
simpleDataSet.RowData.Add("salarySort", (job.Salary.MinimumSalary?.ToString("0000000.00").Replace(".", string.Empty) ?? String.Empty) + " " + (job.Salary.MaximumSalary?.ToString("0000000.00").Replace(".", string.Empty) ?? String.Empty) + " " + (StopWordsRemover != null ? StopWordsRemover.Filter(job.Salary.SalaryRange) : job.Salary.SalaryRange));
simpleDataSet.RowData.Add("hourlyRate", job.Salary.MinimumHourlyRate?.ToString(CultureInfo.CurrentCulture));
simpleDataSet.RowData.Add("hoursPerWeek", job.WorkPattern.HoursPerWeek?.ToString(CultureInfo.CurrentCulture));
simpleDataSet.RowData.Add("jobType", StopWordsRemover != null ? StopWordsRemover.Filter(job.JobType) : job.JobType);
simpleDataSet.RowData.Add("jobTypeDisplay", job.JobType);
simpleDataSet.RowData.Add("contractType", StopWordsRemover != null ? StopWordsRemover.Filter(job.ContractType) : job.ContractType);
simpleDataSet.RowData.Add("department", StopWordsRemover != null ? StopWordsRemover.Filter(job.Department) : job.Department);
simpleDataSet.RowData.Add("departmentDisplay", job.Department);
simpleDataSet.RowData.Add("datePublished", job.DatePublished.ToIso8601DateTime());
if (job.ClosingDate.HasValue)
{
simpleDataSet.RowData.Add("closingDate", job.ClosingDate.Value.ToIso8601DateTime());
simpleDataSet.RowData.Add("closingDateDisplay", job.ClosingDate.Value.ToIso8601DateTime());
}
else
{
// Examine queries are simpler if there is always a closing date, so set a far-future closing date to represent never,
// and don't have a version of the closing date for display.
simpleDataSet.RowData.Add("closingDate", DateTime.MaxValue.ToIso8601DateTime());
}
var workPatternList = string.Join(", ", job.WorkPattern.WorkPatterns.ToArray <string>());
simpleDataSet.RowData.Add("workPattern", workPatternList);
var locationsList = string.Join(", ", job.Locations.ToArray <string>());
simpleDataSet.RowData.Add("location", StopWordsRemover != null ? StopWordsRemover.Filter(locationsList) : locationsList);
simpleDataSet.RowData.Add("locationDisplay", locationsList); // because Somewhere-on-Sea needs to lose the "on" for searching but keep it for display
if (job.AdvertHtml != null)
{
var fullText = job.AdvertHtml.ToHtmlString();
if (TagSanitiser != null)
{
fullText = TagSanitiser.StripTags(fullText);
}
// Append other fields as keywords, otherwise a search term that's a good match will not be found if it has terms from two fields,
// eg Job Title (full time)
const string space = " ";
fullText = new StringBuilder(fullText)
.Append(space).Append(job.Reference)
.Append(space).Append(job.JobTitle)
.Append(space).Append(job.Organisation)
.Append(space).Append(job.Locations)
.Append(space).Append(job.JobType)
.Append(space).Append(job.ContractType)
.Append(space).Append(job.Department)
.Append(space).Append(job.WorkPattern.ToString())
.ToString();
simpleDataSet.RowData.Add("fullText", fullText);
simpleDataSet.RowData.Add("fullHtml", job.AdvertHtml.ToHtmlString());
}
if (job.AdditionalInformationHtml != null)
{
simpleDataSet.RowData.Add("additionalInfo", job.AdditionalInformationHtml.ToHtmlString());
}
if (job.EqualOpportunitiesHtml != null)
{
simpleDataSet.RowData.Add("equalOpportunities", job.EqualOpportunitiesHtml.ToHtmlString());
}
simpleDataSet.RowData.Add("applyUrl", job.ApplyUrl?.ToString());
return(simpleDataSet);
}
/// <summary>
/// Add all YAML fields to the index
/// </summary>
/// <param name="simpleDataSet"></param>
/// <param name="lines"></param>
/// <returns>The linenumber of the second YAML marker</returns>
private static int AddYamlFields(SimpleDataSet simpleDataSet, List <string> lines)
{
// Check if the first line is a YAML marker
// YAML is only accepted if it's on the top of the document
// because empty lines are already removed, the index needs to be 0
bool hasYaml = lines.ElementAt(0).TrimEnd() == "---";
int secondYamlMarker = 0;
if (hasYaml)
{
// Find the "next" triple dash starting from the second line
// But first trim all trailing spaces as this only creates issues which are hard to debug
// and unclear for users. Make sure you have a ToList because IEnumerable has no IndexOf()
secondYamlMarker = lines
.Select(l => l.TrimEnd())
.ToList()
.IndexOf("---", 1);
// add all yaml together and parse YAML meta data
YamlMetaData yamlMetaData = new YamlMetaData();
if (secondYamlMarker > 0)
{
// we found a second marker, so we have YAML data available
var yamlInput = new StringBuilder();
for (int i = 1; i < secondYamlMarker; i++)
{
yamlInput.AppendLine(lines.ElementAt(i));
}
;
// Try to convert the YAML text to a strongly typed model using YamlDotNet
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention())
.IgnoreUnmatchedProperties()
.Build();
yamlMetaData = deserializer.Deserialize <YamlMetaData>(yamlInput.ToString());
}
// Add Yaml stuff to the LUCENE index
simpleDataSet.RowData.Add("tags", yamlMetaData.Tags);
simpleDataSet.RowData.Add("keywords", yamlMetaData.Keywords);
simpleDataSet.RowData.Add("versionFrom", yamlMetaData.VersionFrom);
simpleDataSet.RowData.Add("versionTo", yamlMetaData.VersionTo);
simpleDataSet.RowData.Add("assetID", yamlMetaData.AssetId);
simpleDataSet.RowData.Add("product", yamlMetaData.Product);
simpleDataSet.RowData.Add("topics", yamlMetaData.Topics);
simpleDataSet.RowData.Add("audience", yamlMetaData.Topics);
simpleDataSet.RowData.Add("complexity", yamlMetaData.Complexity);
simpleDataSet.RowData.Add("meta.Title", yamlMetaData.MetaTitle);
simpleDataSet.RowData.Add("meta.Description", yamlMetaData.MetaDescription);
simpleDataSet.RowData.Add("versionRemoved", yamlMetaData.VersionRemoved);
simpleDataSet.RowData.Add("needsV8Update", yamlMetaData.NeedsV8Update);
var matchingMajorVersions = CalculateMajorVersions(yamlMetaData);
simpleDataSet.RowData.Add("majorVersion", string.Join(" ", matchingMajorVersions));
}
else
{
// no YAML information, add the current version as majorVersion
simpleDataSet.RowData.Add("majorVersion", GetCurrentDocVersion().ToString());
}
return(secondYamlMarker);
}
internal SimpleDataSet IndexLocation(EditableLocation Location, string IndexType, int IndexNodeId)
{
//LogHelper.Info<LocationIndexManager>(string.Format("IndexLocation for {0} (#{1}) STARTED", Location.Name, IndexNodeId));
// create the node definition, ensure that it is the same type as referenced in the config
var sds = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
sds.NodeDefinition.NodeId = IndexNodeId;
sds.NodeDefinition.Type = IndexType;
// add the data to the row (These are all listed in the ExamineIndex.config file)
sds.RowData.Add("Key", Location.Key.ToString());
sds.RowData.Add("Name", Location.Name);
sds.RowData.Add("LocationTypeName", Location.LocationType.Name);
sds.RowData.Add("LocationTypeKey", Location.LocationTypeKey.ToString());
sds.RowData.Add("Latitude", Location.Latitude.ToString());
sds.RowData.Add("Longitude", Location.Longitude.ToString());
sds.RowData.Add("Address1", Location.Address.Address1);
sds.RowData.Add("Address2", Location.Address.Address2);
sds.RowData.Add("Locality", Location.Address.Locality);
sds.RowData.Add("Region", Location.Address.Region);
sds.RowData.Add("PostalCode", Location.Address.PostalCode);
sds.RowData.Add("CountryCode", Location.Address.CountryCode);
sds.RowData.Add("Email", Location.Email);
sds.RowData.Add("Phone", Location.Phone);
//We will add each custom property to the index,
//but in addition, all the custom data will be added in a blob - in case the <IndexUserFields> in ExamineIndex.config hasn't been updated with every custom property
var allCustomPropData = new StringBuilder();
foreach (var prop in Location.PropertyData)
{
if (prop.PropertyAttributes.IsDefaultProp == false)
{
try
{
sds.RowData.Add(prop.PropertyAlias, prop.Value.ToString());
}
catch (Exception ex)
{
var msg = string.Format("IndexLocation for {0} (#{1}) Custom Property Data Error on '{2}'", Location.Name, IndexNodeId, prop.PropertyAlias);
LogHelper.Error <LocationIndexManager>(msg, ex);
}
allCustomPropData.AppendFormat("{0}={1}={2}|", prop.Key, prop.PropertyAlias, prop.Value.ToString());
}
}
sds.RowData.Add("CustomPropertyData", allCustomPropData.ToString());
//Put all the data into a single field for simple keyword searches.
var allData = new StringBuilder();
foreach (var field in sds.RowData)
{
allData.AppendFormat("{0} | ", field.Value.ToString());
}
sds.RowData.Add(this.AllDataFieldName, allData.ToString());
//LogHelper.Info<LocationIndexManager>(string.Format("DATA: {0}", sds.RowData.ToExamineXml(IndexNodeId, IndexType)));
//LogHelper.Info<LocationIndexManager>(string.Format("IndexLocation for {0} (#{1}) COMPLETED", Location.Name, IndexNodeId));
return(sds);
}
public void SetData(SimpleDataSet <float> data)
{
_data = data;
AdjustPoints( );
}
public SimpleDataSet MapProjectToSimpleDataIndexItem(IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
int projectVotes, WikiFile[] files, int downloads, IEnumerable <string> compatVersions)
{
var isLive = project.GetPropertyValue <bool>("projectLive");
var isApproved = project.GetPropertyValue <bool>("approved");
var strictPackageFiles = PackageRepositoryService.GetAllStrictSupportedPackageVersions(files);
simpleDataSet.NodeDefinition.NodeId = project.Id;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("body", project.GetPropertyValue <string>("description"));
simpleDataSet.RowData.Add("nodeName", project.Name);
simpleDataSet.RowData.Add("categoryFolder", project.Parent.Name.ToLowerInvariant().Trim());
simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "project");
simpleDataSet.RowData.Add("url", project.Url);
simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue <string>("packageGuid"));
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue <string>("worksOnUaaS"));
var imageFile = string.Empty;
if (project.HasValue("defaultScreenshotPath"))
{
imageFile = project.GetPropertyValue <string>("defaultScreenshotPath");
}
if (string.IsNullOrWhiteSpace(imageFile))
{
var image = files.FirstOrDefault(x => x.FileType == "screenshot");
if (image != null)
{
imageFile = image.Path;
}
}
//Clean up version data before its included in the index, the reason we have to do this
// is due to the way the version data is stored, you can see it in uVersion.config - it's super strange
// because of the 3 digit nature but when it doesn't end with a '0' it's actually just the major/minor version
// so we have to do all of this parsing.
var version = project.GetPropertyValue <string>("compatibleVersions") ?? string.Empty;
var cleanedVersions = version.ToLower()
.Trim(',')
.Split(',')
.Select(x => x.GetFromUmbracoString(reduceToConfigured: false))
.Where(x => x != null);
var cleanedCompatVersions = compatVersions
.Select(x => x.GetFromUmbracoString(reduceToConfigured: false))
.Where(x => x != null);
//popularity for sorting number = downloads + karma * 100;
//TODO: Change score so that we take into account:
// - recently updated
// - works on latest umbraco versions
// - works on uaas
// - has a forum
// - has source code link
// - open for collab / has collaborators
// - download count in a recent timeframe - since old downloads should count for less
var pop = downloads + (projectVotes * 100);
simpleDataSet.RowData.Add("popularity", pop.ToString());
simpleDataSet.RowData.Add("karma", projectVotes.ToString());
simpleDataSet.RowData.Add("downloads", downloads.ToString());
simpleDataSet.RowData.Add("image", imageFile);
var packageFiles = files.Count(x => x.FileType == "package");
simpleDataSet.RowData.Add("packageFiles", packageFiles.ToString());
simpleDataSet.RowData.Add("projectLive", isLive ? "1" : "0");
simpleDataSet.RowData.Add("approved", isApproved ? "1" : "0");
//now we need to add the versions and compat versions
// first, this is the versions that the project has files tagged against
simpleDataSet.RowData.Add("versions", string.Join(",", cleanedVersions));
//then we index the versions that the project has actually been flagged as compatible against
simpleDataSet.RowData.Add("compatVersions", string.Join(",", cleanedCompatVersions));
simpleDataSet.RowData.Add("minimumVersionStrict", string.Join(",", strictPackageFiles.Select(x => x.MinUmbracoVersion.ToString(3))));
return(simpleDataSet);
}
private void AdjustPoints( )
{
if (_data.Count < 2)
{
return;
}
SimpleDataSet <Vector3> newDataSet = _data.GenerateNewDataSet <Vector3>(
(f, i) => {
return(new Vector3(0.0f, f, 0.0f));
});
float prevY = 0.0f;
float prevZ = 0.0f;
newDataSet.RunOverAllData((d, t, i) => {
if (i == 0)
{
d.z = d.y;
}
else
{
d.y = Mathf.Lerp(d.y, prevY, _smoothing);
d.z = Mathf.Lerp(d.y, prevZ, _smoothing);
}
prevY = d.y;
prevZ = d.z;
d.x = t;
return(d);
});
SimpleDataSet <Vector3> .SimpleDataEntry[] data = newDataSet.GetLastEntries(maxEntries);
Vector3[] adjPoints = new Vector3[data.Length];
for (int i = 0; i < adjPoints.Length; ++i)
{
adjPoints[i] = data[i].Data;
if (_drawTrend)
{
adjPoints[i].y = adjPoints[i].z;
}
adjPoints[i].z = 0.0f;
}
_currXMax = float.NegativeInfinity;
_currYMin = float.PositiveInfinity;
_currYMax = float.NegativeInfinity;
_currYMin = float.PositiveInfinity;
for (int i = 0; i < data.Length; ++i)
{
if (_currXMax < adjPoints[i].x)
{
_currXMax = adjPoints[i].x;
}
if (_currXMin > adjPoints[i].x)
{
_currXMin = adjPoints[i].x;
}
if (_currYMax < adjPoints[i].y)
{
_currYMax = adjPoints[i].y;
}
if (_currYMin > adjPoints[i].y)
{
_currYMin = adjPoints[i].y;
}
}
// create display
for (int i = 0; i < adjPoints.Length; ++i)
{
adjPoints[i].x = Mathf.Lerp(renderArea.xMin, renderArea.xMax, Mathf.InverseLerp(_currXMin, _currXMax, adjPoints[i].x));
adjPoints[i].y = Mathf.Lerp(renderArea.yMin, renderArea.yMax, Mathf.InverseLerp(_currYMin, _currYMax, adjPoints[i].y));
adjPoints[i].z = 0.0f;
}
_line.positionCount = adjPoints.Length;
_line.SetPositions(adjPoints);
}
public static SimpleDataSet MapTopicToSimpleDataIndexItem(ReadOnlyTopic topic, SimpleDataSet simpleDataSet, int id, string indexType)
{
//First generate the accumulated comment text:
var commentText = string.Empty;
foreach (var currentComment in topic.Comments.Where(c => c.IsSpam == false))
{
commentText += currentComment.Body;
}
var body = library.StripHtml(topic.Body + commentText);
simpleDataSet.NodeDefinition.NodeId = id;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("body", body);
if (!string.IsNullOrEmpty(commentText))
{
simpleDataSet.RowData.Add("comments", commentText);
}
simpleDataSet.RowData.Add("nodeName", topic.Title);
simpleDataSet.RowData.Add("updateDate", topic.Updated.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "forum");
simpleDataSet.RowData.Add("urlName", topic.UrlName);
simpleDataSet.RowData.Add("createDate", topic.Created.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("latestCommentId", topic.LatestComment.ToString());
simpleDataSet.RowData.Add("latestReplyAuthorId", topic.LatestReplyAuthor.ToString());
if (!string.IsNullOrEmpty(topic.LastReplyAuthorName))
{
simpleDataSet.RowData.Add("latestReplyAuthorName", topic.LastReplyAuthorName);
}
simpleDataSet.RowData.Add("authorId", topic.MemberId.ToString());
if (!string.IsNullOrEmpty(topic.AuthorName))
{
simpleDataSet.RowData.Add("authorName", topic.AuthorName);
}
simpleDataSet.RowData.Add("parentId", topic.ParentId.ToString());
simpleDataSet.RowData.Add("replies", topic.Replies.ToString());
simpleDataSet.RowData.Add("locked", topic.Locked.ToString());
simpleDataSet.RowData.Add("solved", topic.Answer.ToString());
simpleDataSet.RowData.Add("version", topic.Version.ToString());
return(simpleDataSet);
}
public SimpleDataSet MapProjectToSimpleDataIndexItem(IPublishedContent project, SimpleDataSet simpleDataSet, string indexType,
int projectVotes, WikiFile[] files, int downloads, IEnumerable <string> compatVersions)
{
var isLive = project.GetPropertyValue <bool>("projectLive");
var isApproved = project.GetPropertyValue <bool>("approved");
var minimumVersionStrict = string.Empty;
var currentFileId = project.GetPropertyValue <int>("file");
if (currentFileId > 0)
{
var currentFile = files.FirstOrDefault(x => x.Id == currentFileId);
if (currentFile != null)
{
minimumVersionStrict = currentFile.MinimumVersionStrict;
}
}
simpleDataSet.NodeDefinition.NodeId = project.Id;
simpleDataSet.NodeDefinition.Type = indexType;
simpleDataSet.RowData.Add("body", project.GetPropertyValue <string>("description"));
simpleDataSet.RowData.Add("nodeName", project.Name);
simpleDataSet.RowData.Add("categoryFolder", project.Parent.Name.ToLowerInvariant().Trim());
simpleDataSet.RowData.Add("updateDate", project.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("createDate", project.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"));
simpleDataSet.RowData.Add("nodeTypeAlias", "project");
simpleDataSet.RowData.Add("url", project.Url);
simpleDataSet.RowData.Add("uniqueId", project.GetPropertyValue <string>("packageGuid"));
simpleDataSet.RowData.Add("worksOnUaaS", project.GetPropertyValue <string>("worksOnUaaS"));
simpleDataSet.RowData.Add("minimumVersionStrict", minimumVersionStrict);
var imageFile = string.Empty;
if (project.HasValue("defaultScreenshotPath"))
{
imageFile = project.GetPropertyValue <string>("defaultScreenshotPath");
}
if (string.IsNullOrWhiteSpace(imageFile))
{
var image = files.FirstOrDefault(x => x.FileType == "screenshot");
if (image != null)
{
imageFile = image.Path;
}
}
//Clean up version data before its included in the index, the reason we have to do this
// is due to the way the version data is stored, you can see it in uVersion.config - it's super strange
// because of the 3 digit nature but when it doesn't end with a '0' it's actually just the major/minor version
// so we have to do all of this parsing.
var version = project.GetPropertyValue <string>("compatibleVersions") ?? string.Empty;
var cleanedVersions = version.ToLower()
.Replace("nan", "")
.Replace("saved", "")
.Replace("v", "")
.Trim(',')
.Split(',')
//it's stored as an int like 721 (for version 7.2.1)
.Where(x => x.Length <= 3 && x.Length > 0)
//pad it out to 3 digits
.Select(x => x.PadRight(3, '0'))
.Select(x =>
{
int o;
if (int.TryParse(x, out o))
{
//if it ends with '0', that means it's a X.X.X version
// if it does not end with '0', that means that the last 2 digits are the
// Minor part of the version
return(x.EndsWith("0")
? string.Format("{0}.{1}.{2}", x[0], x[1], 0)
: string.Format("{0}.{1}.{2}", x[0], x.Substring(1), 0));
}
return(null);
})
.Where(x => x != null);
var cleanedCompatVersions = compatVersions.Select(x => x.Replace("nan", "")
.Replace("saved", "")
.Replace("nan", "")
.Replace("v", "")
.Replace(".x", "")
.Trim(','));
//popularity for sorting number = downloads + karma * 100;
//TODO: Change score so that we take into account:
// - recently updated
// - works on latest umbraco versions
// - works on uaas
// - has a forum
// - has source code link
// - open for collab / has collaborators
// - download count in a recent timeframe - since old downloads should count for less
var pop = downloads + (projectVotes * 100);
simpleDataSet.RowData.Add("popularity", pop.ToString());
simpleDataSet.RowData.Add("karma", projectVotes.ToString());
simpleDataSet.RowData.Add("downloads", downloads.ToString());
simpleDataSet.RowData.Add("image", imageFile);
var packageFiles = files.Count(x => x.FileType == "package");
simpleDataSet.RowData.Add("packageFiles", packageFiles.ToString());
simpleDataSet.RowData.Add("projectLive", isLive ? "1" : "0");
simpleDataSet.RowData.Add("approved", isApproved ? "1" : "0");
//now we need to add the versions and compat versions
// first, this is the versions that the project has files tagged against
simpleDataSet.RowData.Add("versions", string.Join(",", cleanedVersions));
//then we index the versions that the project has actually been flagged as compatible against
simpleDataSet.RowData.Add("compatVersions", string.Join(",", cleanedCompatVersions));
return(simpleDataSet);
}