public IEnumerable <SimpleDataSet> GetAllData(string indexType)
{
var umbContxt = EnsureUmbracoContext();
var projects = umbContxt.ContentCache.GetByXPath("//Community/Projects//Project [projectLive='1']").ToArray();
var nugetService = new NugetPackageDownloadService();
var nugetDownloads = nugetService.GetNugetPackageDownloads();
var allProjectIds = projects.Select(x => x.Id).ToArray();
var allProjectKarma = Utils.GetProjectTotalVotes();
var allProjectWikiFiles = WikiFile.CurrentFiles(allProjectIds);
var allProjectDownloads = Utils.GetProjectTotalPackageDownload();
var allCompatVersions = Utils.GetProjectCompatibleVersions();
var mostRecentDownloadDate = WikiFile.GetMostRecentDownloadDate();
// if most recent download date is MinValue then there is no download data to query
var downloadStats = mostRecentDownloadDate == DateTime.MinValue
? new Dictionary <int, MonthlyProjectDownloads>()
: WikiFile.GetMonthlyDownloadStatsByProject(mostRecentDownloadDate.Subtract(TimeSpan.FromDays(365)));
foreach (var project in projects)
{
LogHelper.Debug(this.GetType(), "Indexing " + project.Name);
var simpleDataSet = new SimpleDataSet {
NodeDefinition = new IndexedNode(), RowData = new Dictionary <string, string>()
};
var projectDownloads = allProjectDownloads.ContainsKey(project.Id) ? allProjectDownloads[project.Id] : 0;
var projectKarma = allProjectKarma.ContainsKey(project.Id) ? allProjectKarma[project.Id] : 0;
var projectFiles = allProjectWikiFiles.ContainsKey(project.Id) ? allProjectWikiFiles[project.Id].ToArray() : new WikiFile[] { };
var projectVersions = allCompatVersions.ContainsKey(project.Id) ? allCompatVersions[project.Id] : Enumerable.Empty <string>();
var nugetPackageId = nugetService.GetNuGetPackageId(project);
int?dailyNugetDownLoads = null;
if (!string.IsNullOrWhiteSpace(nugetPackageId))
{
var packageInfo = nugetDownloads.FirstOrDefault(x => x.PackageId == nugetPackageId);
if (packageInfo != null)
{
projectDownloads += packageInfo.TotalDownLoads;
dailyNugetDownLoads = packageInfo.AverageDownloadPerDay;
}
}
yield return(MapProjectToSimpleDataIndexItem(
downloadStats,
mostRecentDownloadDate,
project, simpleDataSet, indexType, projectKarma, projectFiles, projectDownloads, projectVersions, dailyNugetDownLoads));
}
}
private Models.Package MapContentToPackage(IPublishedContent content, SearchResult result)
{
if (content == null)
{
return(null);
}
var ownerId = content != null& content.HasProperty("owner") ? content.GetPropertyValue <int>("owner") : 0;
var openForCollab = content != null && content.HasProperty("openForCollab") ? content.GetPropertyValue <bool>("openForCollab", false) : false;
var score = result != null?GetScore(result.Fields) : 0;
var version = result != null?ParseVersion(result) : "";
var downloads = result != null?GetCombinedDownloads(result.Fields, Utils.GetProjectTotalDownloadCount(content.Id)) : Utils.GetProjectTotalDownloadCount(content.Id);
var nugetService = new NugetPackageDownloadService();
var nuGetPackageId = nugetService.GetNuGetPackageId(content);
return(new Models.Package
{
Category = content.Parent.Name,
Created = content.CreateDate,
Excerpt = GetPackageExcerpt(content, 12),
Downloads = downloads,
Id = content.GetPropertyValue <Guid>("packageGuid"),
Likes = Utils.GetProjectTotalVotes(content.Id),
Name = content.Name,
Icon = GetThumbnailUrl(_baseUrl + content.GetPropertyValue <string>("defaultScreenshotPath", "/css/img/package2.png"), 154, 281),
LatestVersion = content.GetPropertyValue <string>("version"),
OwnerInfo = ownerId != 0 ? GetPackageOwnerInfo(ownerId, openForCollab, content.Id) : new PackageOwnerInfo(),
Url = string.Concat(_baseUrl, content.Url),
// stuff added to combine the search between our.umbraco.com and the backoffice
Score = score,
VersionRange = version,
Image = _baseUrl + content.GetPropertyValue <string>("defaultScreenshotPath", "/css/img/package2.png"),
Summary = GetPackageSummary(content, 50),
CertifiedToWorkOnUmbracoCloud = content.GetPropertyValue <bool>("worksOnUaaS"),
NuGetPackageId = nuGetPackageId,
IsNuGetFormat = content.GetPropertyValue <bool>("isNuGetFormat"),
IsPromoted = content.GetPropertyValue <bool>("isPromoted")
});
}
/// <summary>
/// Returns a PackageDetails instance
/// </summary>
/// <param name="content"></param>
/// <param name="currentUmbracoVersion"></param>
/// <returns>
/// If the current umbraco version is not compatible with any package files, the ZipUrl and ZipFileId will be empty
/// </returns>
private PackageDetails MapContentToPackageDetails(IPublishedContent content, System.Version currentUmbracoVersion)
{
if (currentUmbracoVersion == null)
{
throw new ArgumentNullException("currentUmbracoVersion");
}
if (content == null)
{
return(null);
}
var package = MapContentToPackage(content, null);
if (package == null)
{
return(null);
}
var wikiFiles = WikiFile.CurrentFiles(content.Id);
//TODO: SD: I dunno where these come from or if we care about it?
//var deliCompatVersions = Utils.GetProjectCompatibleVersions(content.Id) ?? new List<string>();
var allPackageFiles = wikiFiles.Where(x => x.FileType.InvariantEquals("package") && x.Archived == false).ToArray();
//get the strict packages in the correct desc order
var strictPackageFileVersions = GetAllStrictSupportedPackageVersions(allPackageFiles).ToArray();
//these are ordered by package version desc
var nonStrictPackageFiles = GetNonStrictSupportedPackageVersions(allPackageFiles).ToArray();
var nugetService = new NugetPackageDownloadService();
var nuGetPackageId = nugetService.GetNuGetPackageId(content);
var packageDetails = new PackageDetails(package)
{
TargetedUmbracoVersions = GetAllFilePackageVersions(allPackageFiles).Select(x => {
//ensure the version has consistent parts (major.minor.build)
var version = x.Build >= 0 ? x : new System.Version(x.Major, x.Minor, 0);
return(version.ToString(3));
}).ToArray(),
Compatibility = GetPackageCompatibility(content),
StrictFileVersions = strictPackageFileVersions.Select(x => new PackageFileVersion
{
PackageVersion = x.PackageVersion.Build >= 0
? x.PackageVersion.ToString(3)
: new System.Version(x.PackageVersion.Major, x.PackageVersion.Minor, 0).ToString(3),
MinUmbracoVersion = x.MinUmbracoVersion.Build >= 0
? x.MinUmbracoVersion.ToString(3)
: new System.Version(x.MinUmbracoVersion.Major, x.MinUmbracoVersion.Minor, 0).ToString(3),
FileId = x.FileId
}).ToList(),
NetVersion = content.GetPropertyValue <string>("dotNetVersion"),
LicenseName = content.GetPropertyValue <string>("licenseName"),
LicenseUrl = content.GetPropertyValue <string>("licenseUrl"),
Description = content.GetPropertyValue <string>("description").CleanHtmlAttributes(),
Images = GetPackageImages(wikiFiles.Where(x => x.FileType.InvariantEquals("screenshot")), 154, 281),
ExternalSources = GetExternalSources(content),
NuGetPackageId = nuGetPackageId,
IsNuGetFormat = content.GetPropertyValue <bool>("isNuGetFormat")
};
var version75 = new System.Version(7, 5, 0);
//this is the file marked as the current/latest release
var currentReleaseFile = content.GetPropertyValue <int>("file");
if (strictPackageFileVersions.Length == 0)
{
//if there are no strict package files then return the latest package file
packageDetails.ZipUrl = string.Concat(BASE_URL, "/FileDownload?id=", currentReleaseFile);
packageDetails.ZipFileId = currentReleaseFile;
}
else if (currentUmbracoVersion < version75)
{
//if the umbraco version is < 7.5 it means that strict package formats are not supported
AssignLatestNonStrictPackageFile(nonStrictPackageFiles, currentReleaseFile, packageDetails);
}
else
{
//this package has some strict version dependency files, so we need to figure out which one is
// compatible with the current umbraco version passed in and also use the latest available
// package version that is compatible.
int found = -1;
foreach (var pckVersion in strictPackageFileVersions)
{
//if this version will work with the umbraco version, then use it
if (currentUmbracoVersion >= pckVersion.MinUmbracoVersion)
{
found = pckVersion.FileId;
break;
}
}
if (found != -1)
{
//got one! so use it's id for the file download
packageDetails.ZipUrl = string.Concat(BASE_URL, "/FileDownload?id=", found);
packageDetails.ZipFileId = found;
}
else if (nonStrictPackageFiles.Length > 0)
{
//Here's the other case, if this package has both strict and non-strict package file versions and we didn't find one above,
//than we need to determine if the latest non-strict package file format should be used for the current version being passed in
AssignLatestNonStrictPackageFile(nonStrictPackageFiles, currentReleaseFile, packageDetails);
}
}
packageDetails.Created = content.CreateDate;
return(packageDetails);
}
public void GetNugetDownloads(PerformContext content)
{
var nugetService = new NugetPackageDownloadService();
RecurringJob.AddOrUpdate(() => nugetService.ImportNugetPackageDownloads(), Cron.Daily);
}
public ActionResult GetPackageInfos()
{
var packages = GetAllPackages();
var packageInfos = new List <PackageInfo>();
var nugetService = new NugetPackageDownloadService();
var nugetPackageDownloads = nugetService.GetNugetPackageDownloads();
var licenses = new HashSet <string>();
foreach (var package in packages)
{
var compatibleVersions = package.GetPropertyValue <string>("compatibleVersions");
var categoryName = string.Empty;
var categoryNodeId = package.GetPropertyValue <int>("category");
if (categoryNodeId != 0)
{
categoryName = Umbraco.TypedContent(categoryNodeId).Name;
}
var ownerName = string.Empty;
var ownerMemberId = package.GetPropertyValue <int>("owner");
if (ownerMemberId != 0)
{
ownerName = Umbraco.TypedMember(ownerMemberId).Name;
}
var ourDownloads = Utils.GetProjectTotalDownloadCount(package.Id);
var nugetPackageId = nugetService.GetNuGetPackageId(package);
var nugetDownloads = 0;
var nugetPackageInfo = nugetPackageDownloads.FirstOrDefault(x => x.PackageId == nugetPackageId);
if (nugetPackageInfo != null)
{
nugetDownloads = nugetPackageInfo.TotalDownLoads;
}
var lastUpdate = package.UpdateDate;
// If files have been updated later than this then use that date
var wikiFiles = WikiFile.CurrentFiles(package.Id);
var latestFile = wikiFiles.Where(x => x.FileType == "package").OrderByDescending(x => x.CreateDate).FirstOrDefault();
if (latestFile != null && latestFile.CreateDate > lastUpdate)
{
lastUpdate = latestFile.CreateDate;
}
var licenseName = package.GetPropertyValue <string>("licenseName");
licenses.Add(licenseName);
var openSource = false;
var openSourceChecks = new List <string>
{
"MIT",
"CC",
"BSD",
"Apache",
"GPL",
"GNU",
"Free",
"Unlicense",
"MS-PL",
"OSL",
"MPL",
"Open Government",
"WTFPL",
// Questionable but.. ok
"TBD",
"Undefined",
"No license"
};
// These two match the above, but are not open source
if (licenseName != "Resizer Freedom license" && licenseName != "MIT + Commercial")
{
foreach (var item in openSourceChecks)
{
if (!licenseName.InvariantContains(item))
{
continue;
}
openSource = true;
// Immediately break out of foreach, no more checks needed
break;
}
}
var packageInfo = new PackageInfo
{
Name = package.Name,
Category = categoryName,
// We don't know.. look at the license maybe?
OpenSource = openSource,
License = licenseName,
LicenseUrl = package.GetPropertyValue <string>("licenseUrl"),
Owner = ownerName,
CloudCompatible = package.GetPropertyValue <bool>("worksOnUaaS"),
DownloadsTotal = ourDownloads + nugetDownloads,
LastUpdate = lastUpdate.ToString("yyyy-MM-dd"),
Version7Compatible = compatibleVersions.InvariantContains("v7"),
Version8Compatible = compatibleVersions.InvariantContains("v8"),
Url = package.UrlWithDomain()
};
packageInfos.Add(packageInfo);
}
var json = JsonConvert.SerializeObject(packageInfos);
var dt = (DataTable)JsonConvert.DeserializeObject(json, typeof(DataTable));
var csv = DataTableToCsv(dt);
var fileBytes = Encoding.UTF8.GetBytes(string.Join(Environment.NewLine, csv));
var file = File(fileBytes, "text/csv", "packaginfos.csv");
return(file);
}