// Returns list of 2-element arrays, element 0 is field name (or null), element 1 is term/phrase
public List<NuGetSearchTerm> Parse(string searchTerm)
{
var terms = new List<NuGetSearchTerm>();
_tokenizer = new Tokenizer(searchTerm);
while (_tokenizer.Peek() != TokenType.Eof)
{
var term = new NuGetSearchTerm();
if (_tokenizer.Peek() == TokenType.Field)
{
if (ParseField(term))
{
terms.Add(term);
}
}
else
{
if (ParseTermOrPhrase(term))
{
terms.Add(term);
}
}
}
return terms;
}
public override void ExecuteCommand()
{
sqlQueryForDbAge = string.Format("select create_date from sys.databases where name = '{0}'", DbName);
thresholdValues = new JavaScriptSerializer().Deserialize<AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
List<Tuple<string, string>> jobOutputs = new List<Tuple<string, string>>();
jobOutputs.Add(new Tuple<string,string>("SyncPackagesToFailoverDC", CheckLagBetweenDBAndBlob()));
jobOutputs.Add(new Tuple<string, string>("ImportCompletionStatus", CheckForInCompleteDBImport()));
JArray reportObject = ReportHelpers.GetJson(jobOutputs);
ReportHelpers.CreateBlob(StorageAccount, "RunBackgroundCheckForFailoverDCReport.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
public static IQueryable<Package> Search(this IQueryable<Package> source, string searchTerm)
{
if (String.IsNullOrWhiteSpace(searchTerm))
{
return source;
}
// Split the search terms by spaces
var terms = (searchTerm ?? String.Empty).Split();
// Build a list of expressions for each term
var expressions = new List<LambdaExpression>();
foreach (var term in terms)
{
var localSearchTerm = term.to_lower();
if (localSearchTerm.StartsWith("id:"))
{
expressions.Add(idCriteria(localSearchTerm.Replace("id:", string.Empty)));
}
if (localSearchTerm.StartsWith("author:"))
{
expressions.Add(authorCriteria(localSearchTerm.Replace("author:", string.Empty)));
}
else if (localSearchTerm.StartsWith("tag:"))
{
expressions.Add(tagCriteria(localSearchTerm.Replace("tag:", string.Empty)));
} else
{
foreach (var criteria in searchCriteria)
{
expressions.Add(criteria(localSearchTerm));
}
}
}
//todo this becomes an AND
// Build a giant or statement using the bodies of the lambdas
var body = expressions.Select(p => p.Body)
.Aggregate(Expression.OrElse);
// Now build the final predicate
var parameterExpr = Expression.Parameter(typeof(Package));
// Fix up the body to use our parameter expression
body = new ParameterExpressionReplacer(parameterExpr).Visit(body);
// Build the final predicate
var predicate = Expression.Lambda<Func<Package, bool>>(body, parameterExpr);
// Apply it to the query
return source.Where(predicate);
}
public static IQueryable<Package> Search(this IQueryable<Package> source, string searchTerm)
{
// Split the search terms by spaces
var terms = (searchTerm ?? String.Empty).Split();
// Build a list of expressions for each term
var expressions = new List<LambdaExpression>();
foreach (var criteria in searchCriteria)
{
foreach (var term in terms)
{
expressions.Add(criteria(term));
}
}
// Build a giant or statement using the bodies of the lambdas
var body = expressions.Select(p => p.Body)
.Aggregate(Expression.OrElse);
// Now build the final predicate
var parameterExpr = Expression.Parameter(typeof(Package));
// Fix up the body to use our parameter expression
body = new ParameterExpressionReplacer(parameterExpr).Visit(body);
// Build the final predicate
var predicate = Expression.Lambda<Func<Package, bool>>(body, parameterExpr);
// Apply it to the query
return source.Where(predicate);
}
public ActionResult Feed(int? page, int? pageSize)
{
var siteRoot = EnsureTrailingSlash(Configuration.GetSiteRoot(useHttps:false));
IQueryable<Package> packageVersions = packageSvc
.GetPackagesForListing(includePrerelease: false)
.OrderByDescending(p => p.Published);
SyndicationFeed feed = new SyndicationFeed("Chocolatey", "Chocolatey Packages", new Uri(siteRoot));
feed.Copyright = new TextSyndicationContent("Chocolatey copyright FerventCoder Software, Packages copyright original owner(s), Products copyright original author(s).");
feed.Language = "en-US";
List<SyndicationItem> items = new List<SyndicationItem>();
foreach (Package package in packageVersions.ToList())
{
string title = string.Format("{0} ({1})", package.PackageRegistration.Id, package.Version);
var galleryUrl = siteRoot + "packages/" + package.PackageRegistration.Id + "/" + package.Version;
SyndicationItem item = new SyndicationItem(
title,
package.Summary,
new Uri(galleryUrl),
package.PackageRegistration.Id + "." + package.Version,
package.Published
);
item.PublishDate = package.Published;
items.Add(item);
}
//this will bomb until you have one published package in the feed.
feed.LastUpdatedTime = packageVersions.First().Published;
feed.Items = items;
return new RSSActionResult { Feed = feed };
}
public override void ExecuteCommand()
{
thresholdValues = new JavaScriptSerializer().Deserialize<AlertThresholds>(ReportHelpers.Load(StorageAccount, "Configuration.AlertThresholds.json", ContainerName));
List<Tuple<string, string>> jobOutputs = new List<Tuple<string, string>>();
jobOutputs.Add(new Tuple<string, string>("PackageStatics", CheckoutForPackageStatics()));
//jobOutputs.Add(new Tuple<string, string>("PurgePackageStatistics", CheckForPurgePackagStatisticsJob()));
jobOutputs.Add(new Tuple<string, string>("HandleQueuedPackageEdits", CheckForHandleQueuedPackageEditJob()));
// jobOutputs.Add(new Tuple<string, string>("BackupPackages", CheckForBackupPackagesJob())); commenting out this check temporarily as ListBlobs on ng-backups container is giving error.
JArray reportObject = ReportHelpers.GetJson(jobOutputs);
ReportHelpers.CreateBlob(StorageAccount, "RunBackGroundChecksForWorkerJobsReport.json", ContainerName, "application/json", ReportHelpers.ToStream(reportObject));
}
public ActionResult Feed(int? page, int? pageSize)
{
var siteRoot = EnsureTrailingSlash(Configuration.GetSiteRoot(useHttps: false));
IQueryable<Package> packageVersions = packageSvc
.GetPackagesForListing(includePrerelease: false)
.OrderByDescending(p => p.Published);
if (page != null && pageSize != null)
{
int skip = page.GetValueOrDefault() * pageSize.GetValueOrDefault(1);
packageVersions = packageVersions.Skip(skip).Take(pageSize.GetValueOrDefault(1));
}
else if (pageSize != null)
{
packageVersions = packageVersions.Take(pageSize.GetValueOrDefault(1));
}
SyndicationFeed feed = new SyndicationFeed("Chocolatey", "Chocolatey Packages", new Uri(siteRoot));
feed.Copyright = new TextSyndicationContent("Chocolatey copyright RealDimensions Software, LLC, Packages copyright original maintainer(s), Products copyright original author(s).");
feed.Language = "en-US";
List<SyndicationItem> items = new List<SyndicationItem>();
foreach (Package package in packageVersions.ToList().OrEmptyListIfNull())
{
string title = string.Format("{0} ({1})", package.PackageRegistration.Id, package.Version);
var galleryUrl = siteRoot + "packages/" + package.PackageRegistration.Id + "/" + package.Version;
SyndicationItem item = new SyndicationItem(
title,
package.Summary,
new Uri(galleryUrl),
package.PackageRegistration.Id + "." + package.Version,
package.Published
);
item.PublishDate = package.Published;
items.Add(item);
}
try
{
var mostRecentPackage = packageVersions.FirstOrDefault();
feed.LastUpdatedTime = mostRecentPackage == null ? DateTime.Now : mostRecentPackage.Published;
}
catch (Exception)
{
feed.LastUpdatedTime = DateTime.Now;
}
feed.Items = items;
return new RSSActionResult { Feed = feed };
}
public static IDisposable All(params IDisposable[] tokens)
{
return new DisposableAction(() =>
{
var exceptions = new List<Exception>();
foreach (var token in tokens)
{
try
{
token.Dispose();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
}
if (exceptions.Any())
{
throw new AggregateException(exceptions);
}
});
}
Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, IPackage nugetPackage)
{
var package = packageRegistration.Packages
.Where(pv => pv.Version == nugetPackage.Version.ToString())
.SingleOrDefault();
if (package != null)
throw new EntityException("A package with identifier '{0}' and version '{1}' already exists.", packageRegistration.Id, package.Version);
var now = DateTime.UtcNow;
var packageFileStream = nugetPackage.GetStream();
package = new Package
{
Version = nugetPackage.Version.ToString(),
Description = nugetPackage.Description,
ReleaseNotes = nugetPackage.ReleaseNotes,
RequiresLicenseAcceptance = nugetPackage.RequireLicenseAcceptance,
HashAlgorithm = Constants.Sha512HashAlgorithmId,
Hash = cryptoSvc.GenerateHash(packageFileStream.ReadAllBytes()),
PackageFileSize = packageFileStream.Length,
Created = now,
Language = nugetPackage.Language,
LastUpdated = now,
Published = now,
Copyright = nugetPackage.Copyright,
IsPrerelease = !nugetPackage.IsReleaseVersion(),
Listed = true,
};
if (nugetPackage.IconUrl != null)
package.IconUrl = nugetPackage.IconUrl.ToString();
if (nugetPackage.LicenseUrl != null)
package.LicenseUrl = nugetPackage.LicenseUrl.ToString();
if (nugetPackage.ProjectUrl != null)
package.ProjectUrl = nugetPackage.ProjectUrl.ToString();
if (nugetPackage.Summary != null)
package.Summary = nugetPackage.Summary;
if (nugetPackage.Tags != null)
package.Tags = nugetPackage.Tags;
if (nugetPackage.Title != null)
package.Title = nugetPackage.Title;
foreach (var author in nugetPackage.Authors)
package.Authors.Add(new PackageAuthor { Name = author });
var supportedFrameworks = GetSupportedFrameworks(nugetPackage).Select(fn => fn.ToShortNameOrNull()).ToArray();
if (!supportedFrameworks.AnySafe(sf => sf == null))
foreach (var supportedFramework in supportedFrameworks)
package.SupportedFrameworks.Add(new PackageFramework { TargetFramework = supportedFramework });
foreach (var dependencySet in nugetPackage.DependencySets)
{
if (dependencySet.Dependencies.Count == 0)
package.Dependencies.Add(new PackageDependency
{
Id = null,
VersionSpec = null,
TargetFramework = dependencySet.TargetFramework.ToShortNameOrNull()
});
else
foreach (var dependency in dependencySet.Dependencies.Select(d => new { d.Id, d.VersionSpec, dependencySet.TargetFramework }))
package.Dependencies.Add(new PackageDependency
{
Id = dependency.Id,
VersionSpec = dependency.VersionSpec == null ? null : dependency.VersionSpec.ToString(),
TargetFramework = dependency.TargetFramework.ToShortNameOrNull()
});
}
foreach (var packageFile in nugetPackage.GetFiles().OrEmptyListIfNull())
{
var filePath = packageFile.Path;
var fileContent = " ";
IList<string> extensions = new List<string>();
var approvedExtensions = Configuration.ReadAppSettings("PackageFileTextExtensions");
if (!string.IsNullOrWhiteSpace(approvedExtensions))
{
foreach (var extension in approvedExtensions.Split(',', ';'))
{
extensions.Add("." + extension);
}
}
try
{
if (extensions.Contains(Path.GetExtension(filePath)))
{
fileContent = packageFile.GetStream().ReadToEnd();
}
}
catch (Exception ex)
{
// Log but swallow the exception
Elmah.ErrorSignal.FromCurrentContext().Raise(ex);
}
package.Files.Add(new PackageFile
{
FilePath = filePath,
FileContent = fileContent,
});
}
package.FlattenedAuthors = package.Authors.Flatten();
package.FlattenedDependencies = package.Dependencies.Flatten();
return package;
}
public List<ElmahError> GetElmahError(DateTime start, DateTime end)
{
if (StorageAccount == null) StorageAccount = CloudStorageAccount.Parse(ConnectionString);
List<string> nonCriticalErrorDictionary = new JavaScriptSerializer().Deserialize<List<string>>(Load(StorageAccount, "Configuration.ElmahNonCriticalErrors.json", ContainerName));
TableErrorLog log = new TableErrorLog(string.Format(ElmahAccountCredentials));
List<ErrorLogEntry> entities = new List<ErrorLogEntry>();
int lasthours = DateTime.Now.Subtract(start).Hours + 1;
log.GetErrors(0, 500 * lasthours, entities); //retrieve n * LastNHours errors assuming a max of 500 errors per hour.
List<ElmahError> listOfErrors = new List<ElmahError>();
//Get the error from Last N hours.
if (entities.Any(entity => entity.Error.Time.ToUniversalTime() > start.ToUniversalTime() && entity.Error.Time.ToUniversalTime() < end.ToUniversalTime()))
{
entities = entities.Where(entity => entity.Error.Time.ToUniversalTime() > start.ToUniversalTime() && entity.Error.Time.ToUniversalTime() < end.ToUniversalTime()).ToList();
var elmahGroups = entities.GroupBy(item => item.Error.Message);
//Group the error based on exception and send alerts if critical errors exceed the thresold values.
foreach (IGrouping<string, ErrorLogEntry> errorGroups in elmahGroups)
{
Console.WriteLine(errorGroups.Key.ToString() + " " + errorGroups.Count());
int severity = 0;
if (nonCriticalErrorDictionary.Any(item => errorGroups.Key.ToString().Contains(item)))
{
severity = 1; //sev 1 is low pri and sev 0 is high pri.
}
string link = "https://www.nuget.org/Admin/Errors.axd/detail?id={0}";
if (ContainerName.Contains("qa"))
{
link = "https://int.nugettest.org/Admin/Errors.axd/detail?id={0}";
}
//for severity, assume all refresh error, severity = 0
listOfErrors.Add(new ElmahError(errorGroups.Key.ToString(), errorGroups.Count(), errorGroups.Min(item => item.Error.Time.ToLocalTime()), errorGroups.Max(item => item.Error.Time.ToLocalTime()), string.Format(link, errorGroups.First().Id), errorGroups.First().Error.Detail, severity));
}
}
return listOfErrors;
}
/// <summary>
/// Creates report for count and avg time taken for individual scenarios.
/// </summary>
/// <param name="info"></param>
private void CreateUriStemDetailedReport(DirectoryInfo info)
{
List<IISRequestDetails> requestDetails = new List<IISRequestDetails>();
var content = ReportHelpers.Load(StorageAccount,"Configration.IISRequestStems.json",ContainerName);
List<IISRequestDetails> UriStems = new List<IISRequestDetails>();
UriStems = new JavaScriptSerializer().Deserialize<List<IISRequestDetails>>(content);
foreach (IISRequestDetails stem in UriStems)
{
int requestCount = GetDataForUriStem(stem.UriStem, "count (*)", info.FullName);
int avgTime = 0 ;
if (requestCount > 0)
{
avgTime = GetDataForUriStem(stem.UriStem, "avg (time-taken)", info.FullName);
}
requestDetails.Add(new IISRequestDetails(stem.ScenarioName, stem.UriStem, avgTime, requestCount));
}
var json = new JavaScriptSerializer().Serialize(requestDetails);
string blobName = "IISRequestDetails" + string.Format("{0:MMdd}", DateTime.Now.AddHours(-1)) + ".json";
ReportHelpers.AppendDatatoBlob(StorageAccount, blobName, new Tuple<string, string>(string.Format("{0:HH:00}", DateTime.Now.AddHours(-1)), json), 50, ContainerName);
}
private List<PackageEntry> GetMissingPackagesFromLuceneAndRegistrationBlob(HashSet<PackageEntry> catalogPackages)
{
List<PackageEntry> missingPackagesFromLucene = new List<PackageEntry>();
foreach (PackageEntry Catalogentry in catalogPackages)
{
try
{
WebRequest request = WebRequest.Create(string.Format("{0}/find?id={1}&version={2}", SearchEndPoint, Catalogentry.Id, Catalogentry.Version));
request.PreAuthenticate = true;
request.Method = "GET";
WebResponse respose = request.GetResponse();
var statusCode = ((HttpWebResponse)respose).StatusCode;
if (statusCode != HttpStatusCode.OK)
{
missingPackagesFromLucene.Add(new PackageEntry(Catalogentry.Id, Catalogentry.Version));
}
//Check if registration blob exists as well for that Id and version.
request = WebRequest.Create(string.Format("{0}/{1}/{2}.json", RegistrationBlobUrl, Catalogentry.Id.ToLowerInvariant(), Catalogentry.Version.ToLowerInvariant()));
request.PreAuthenticate = true;
request.Method = "GET";
respose = request.GetResponse();
statusCode = ((HttpWebResponse)respose).StatusCode;
if (statusCode != HttpStatusCode.OK)
{
missingPackagesFromLucene.Add(new PackageEntry(Catalogentry.Id, Catalogentry.Version));
}
}catch(Exception)
{ //Add it to missing packages even if the server returns 503.
missingPackagesFromLucene.Add(new PackageEntry(Catalogentry.Id, Catalogentry.Version));
}
}
return missingPackagesFromLucene;
}
private async Task WillThrowIfTheNuGetPackageDependenciesIsLongerThanInt16MaxValue()
{
var service = CreateService();
var versionSpec = VersionRange.Parse("[1.0]");
var numDependencies = 5000;
var packageDependencies = new List<NuGet.Packaging.Core.PackageDependency>();
for (int i = 0; i < numDependencies; i++)
{
packageDependencies.Add(new NuGet.Packaging.Core.PackageDependency("dependency" + i, versionSpec));
}
var nugetPackage = CreateNuGetPackage(packageDependencyGroups: new[]
{
new PackageDependencyGroup(
new NuGetFramework("net40"),
packageDependencies),
});
var ex = await Assert.ThrowsAsync<EntityException>(async () => await service.CreatePackageAsync(nugetPackage.Object, new PackageStreamMetadata(), null));
Assert.Equal(String.Format(Strings.NuGetPackagePropertyTooLong, "Dependencies", Int16.MaxValue), ex.Message);
}
private static IList<StatisticsFact> CreateFacts(JObject data)
{
IList<StatisticsFact> facts = new List<StatisticsFact>();
JToken itemsToken;
// Check if the "Items" exist before trying to access them.
if (!data.TryGetValue("Items", out itemsToken))
{
throw new StatisticsReportNotFoundException();
}
foreach (JObject perVersion in data["Items"])
{
string version = (string)perVersion["Version"];
foreach (JObject perClient in perVersion["Items"])
{
var clientName = (string)perClient["ClientName"];
var clientVersion = (string)perClient["ClientVersion"];
var operation = "unknown";
JToken opt;
if (perClient.TryGetValue("Operation", out opt))
{
operation = (string)opt;
}
var downloads = (int)perClient["Downloads"];
facts.Add(new StatisticsFact(CreateDimensions(version, clientName, clientVersion, operation), downloads));
}
}
return facts;
}
private void CreateUserAgentReport(DirectoryInfo info)
{
string standardError = string.Empty;
string standardOutput = string.Empty;
List<IISUserAgentDetails> userAgentDetails = new List<IISUserAgentDetails>();
var content = ReportHelpers.Load(StorageAccount, "Configuration.IISUserAgent.json", ContainerName);
List<IISUserAgentDetails> userAgents = new List<IISUserAgentDetails>();
userAgents = new JavaScriptSerializer().Deserialize<List<IISUserAgentDetails>>(content);
foreach (IISUserAgentDetails agent in userAgents)
{
string query = string.Format(@"select count(*) from {0}\*{1}*.log WHERE cs(User-Agent) LIKE '{2}'", info.FullName, ReportDate, agent.UserAgent);
int requestCount = InvokeLogParserProcessForUserAgent(@"-i:IISW3C -o:CSV " + @"""" + query + @"""" + " -stats:OFF");
int avgTime = 0;
if (requestCount > 0)
{
query = string.Format(@"select avg(time-taken) from {0}\*{1}*.log WHERE cs(User-Agent) LIKE '{2}'", info.FullName, ReportDate, agent.UserAgent);
avgTime = InvokeLogParserProcessForUserAgent(@"-i:IISW3C -o:CSV " + @"""" + query + @"""" + " -stats:OFF");
}
userAgentDetails.Add(new IISUserAgentDetails(agent.UserAgentName, agent.UserAgent, avgTime, requestCount));
}
string blobName = "IISUserAgentDetails" + ReportDate + ".json";
int count = 0;
foreach (IISUserAgentDetails detail in userAgentDetails)
{
var json = new JavaScriptSerializer().Serialize(detail);
ReportHelpers.AppendDatatoBlob(StorageAccount, blobName, new Tuple<string, string>(count.ToString(), json), userAgentDetails.Count, ContainerName);
count++;
}
}
private static void BackgroundJobsPostStart(IAppConfiguration configuration)
{
var indexer = DependencyResolver.Current.GetService<IIndexingService>();
var jobs = new List<IJob>();
if (indexer != null)
{
indexer.RegisterBackgroundJobs(jobs, configuration);
}
if (!configuration.HasWorker)
{
jobs.Add(
new UpdateStatisticsJob(TimeSpan.FromMinutes(5),
() => new EntitiesContext(configuration.SqlConnectionString, readOnly: false),
timeout: TimeSpan.FromMinutes(5)));
}
if (configuration.CollectPerfLogs)
{
jobs.Add(CreateLogFlushJob());
}
if (configuration.StorageType == StorageType.AzureStorage)
{
var cloudDownloadCountService = DependencyResolver.Current.GetService<IDownloadCountService>() as CloudDownloadCountService;
if (cloudDownloadCountService != null)
{
// Perform initial refresh + schedule new refreshes every 15 minutes
HostingEnvironment.QueueBackgroundWorkItem(cancellationToken => cloudDownloadCountService.Refresh());
jobs.Add(new CloudDownloadCountServiceRefreshJob(TimeSpan.FromMinutes(15), cloudDownloadCountService));
}
}
if (jobs.AnySafe())
{
var jobCoordinator = new NuGetJobCoordinator();
_jobManager = new JobManager(jobs, jobCoordinator)
{
RestartSchedulerOnFailure = true
};
_jobManager.Fail(e => ErrorLog.GetDefault(null).Log(new Error(e)));
_jobManager.Start();
}
}
public virtual ActionResult GetScanResults(string apiKey, string id, string version, string sha256Checksum)
{
if (string.IsNullOrWhiteSpace(apiKey)) return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, string.Format(CultureInfo.CurrentCulture, Strings.InvalidApiKey, apiKey));
if (String.IsNullOrEmpty(id) || String.IsNullOrEmpty(version))
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.NotFound, string.Format(CultureInfo.CurrentCulture, Strings.PackageWithIdAndVersionNotFound, id, version));
}
// if (string.IsNullOrWhiteSpace(sha256Checksum)) return new HttpStatusCodeWithBodyResult(HttpStatusCode.BadRequest, "Sha256Checksum is required.");
if (settings.ScanResultsKey.to_lower() != apiKey.to_lower())
{
return new HttpStatusCodeWithBodyResult(HttpStatusCode.Forbidden, "The specified key does not provide the authority to get scan results for packages");
}
var scanResults = new List<PackageScanResult>();
var results = scanSvc.GetResults(id, version, sha256Checksum);
foreach (var result in results.OrEmptyListIfNull())
{
scanResults.Add(new PackageScanResult
{
FileName = result.FileName.to_string(),
Sha256Checksum = result.Sha256Checksum.to_string(),
Positives = result.Positives.to_string(),
TotalScans = result.TotalScans.to_string(),
ScanDetailsUrl = result.ScanDetailsUrl.to_string(),
ScanData = result.ScanData.to_string(),
ScanDate = result.ScanDate.GetValueOrDefault().ToString(CultureInfo.InvariantCulture),
});
}
return new JsonNetResult(scanResults.ToArray());
}
public virtual async Task<ActionResult> Delete(DeletePackagesRequest deletePackagesRequest)
{
var packagesToDelete = new List<Package>();
if (ModelState.IsValid)
{
// Get the packages to delete
foreach (var package in deletePackagesRequest.Packages)
{
var split = package.Split(new[] {'|'}, StringSplitOptions.RemoveEmptyEntries);
if (split.Length == 2)
{
var packageToDelete = _packageService.FindPackageByIdAndVersion(split[0], split[1], allowPrerelease: true);
if (packageToDelete != null)
{
packagesToDelete.Add(packageToDelete);
}
}
}
// Perform delete
if (deletePackagesRequest.SoftDelete)
{
await _packageDeleteService.SoftDeletePackagesAsync(
packagesToDelete, GetCurrentUser(), EnumHelper.GetDescription(deletePackagesRequest.Reason.Value),
deletePackagesRequest.Signature);
}
else
{
await _packageDeleteService.HardDeletePackagesAsync(
packagesToDelete, GetCurrentUser(), EnumHelper.GetDescription(deletePackagesRequest.Reason.Value),
deletePackagesRequest.Signature,
deletePackagesRequest.DeleteEmptyPackageRegistration);
}
// Redirect out
TempData["Message"] =
"We're performing the package delete right now. It may take a while for this change to propagate through our system.";
return Redirect("/");
}
if (!deletePackagesRequest.Packages.Any())
{
return HttpNotFound();
}
var firstPackage = packagesToDelete.First();
return Delete(firstPackage.PackageRegistration.Id, firstPackage.Version);
}
private static ICollection<GalleryPackageDependency> FromJson_PackageDependencies(JArray array)
{
List<GalleryPackageDependency> packageDependencies = new List<GalleryPackageDependency>();
foreach (JObject obj in array)
{
GalleryPackageDependency packageDependency = new GalleryPackageDependency();
packageDependency.Id = obj["Id"].ToString();
packageDependency.VersionSpec = obj["VersionSpec"].ToString();
packageDependency.TargetFramework = obj["TargetFramework"].ToString();
packageDependencies.Add(packageDependency);
}
return packageDependencies;
}
private List<IISResponseTimeDetails> InvokeLogParserProcessForResponseTime(string arguments, int numFields)
{
List<IISResponseTimeDetails> responseTimeDetails = new List<IISResponseTimeDetails>();
Process nugetProcess = new Process();
ProcessStartInfo nugetProcessStartInfo = new ProcessStartInfo(Path.Combine(Environment.CurrentDirectory, "LogParser.exe"));
nugetProcessStartInfo.Arguments = arguments;
nugetProcessStartInfo.RedirectStandardError = true;
nugetProcessStartInfo.RedirectStandardOutput = true;
nugetProcessStartInfo.RedirectStandardInput = true;
nugetProcessStartInfo.UseShellExecute = false;
nugetProcess.StartInfo = nugetProcessStartInfo;
nugetProcess.Start();
StreamReader reader = nugetProcess.StandardOutput;
reader.ReadLine(); //this has the field names, ignore it.
while (!reader.EndOfStream)
{
string responseTimeOutput = reader.ReadLine();
string[] responseTimeOutputs = responseTimeOutput.Split(',');
if (responseTimeOutputs.Count() == numFields)
{
responseTimeDetails.Add(new IISResponseTimeDetails(responseTimeOutputs[0], Convert.ToInt32(responseTimeOutputs[1])));
}
}
nugetProcess.WaitForExit();
List<IISResponseTimeDetails> filteredList = new List<IISResponseTimeDetails>();
filteredList = responseTimeDetails.OrderByDescending(x => x.AvgTimeTakenInMilliSeconds).Take(15).ToList();
return filteredList;
}
private static ICollection<User> FromJson_Owners(JArray array)
{
List<User> owners = new List<User>();
foreach (JToken token in array)
{
User owner = new User();
owner.Username = token.ToString();
owners.Add(owner);
}
return owners;
}
public async Task<StatisticsPackagesReport> GetPackageVersionDownloadsByClient(string packageId, string packageVersion)
{
try
{
if (string.IsNullOrEmpty(packageId) || string.IsNullOrEmpty(packageVersion))
{
return null;
}
var reportName = string.Format(CultureInfo.CurrentCulture, RecentpopularityDetailBlobNameFormat,
StatisticsReportName.RecentPopularityDetail_, packageId).ToLowerInvariant();
var reportContent = await _reportService.Load(reportName);
if (reportContent == null)
{
return null;
}
var content = JObject.Parse(reportContent.Content);
var report = new StatisticsPackagesReport
{
LastUpdatedUtc = reportContent.LastUpdatedUtc
};
var facts = new List<StatisticsFact>();
foreach (var fact in CreateFacts(content))
{
if (fact.Dimensions["Version"] == packageVersion)
{
facts.Add(fact);
}
}
report.Facts = facts;
return report;
}
catch (NullReferenceException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (JsonReaderException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (StorageException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (ArgumentException e)
{
QuietLog.LogHandledException(e);
return null;
}
}
private static ICollection<PackageFramework> FromJson_SupportedFrameworks(JArray array)
{
List<PackageFramework> supportedFrameworks = new List<PackageFramework>();
foreach (JToken token in array)
{
PackageFramework supportedFramework = new PackageFramework();
supportedFramework.TargetFramework = token.ToString();
supportedFrameworks.Add(supportedFramework);
}
return supportedFrameworks;
}
private Package CreatePackageFromNuGetPackage(PackageRegistration packageRegistration, IPackage nugetPackage)
{
var package = packageRegistration.Packages
.Where(pv => pv.Version == nugetPackage.Version.ToString())
.SingleOrDefault();
if (package != null)
{
switch (package.Status)
{
case PackageStatusType.Rejected :
throw new EntityException(
string.Format(
"This package has been {0} and can no longer be submitted.",
package.Status.GetDescriptionOrValue().ToLower()));
case PackageStatusType.Submitted :
//continue on
break;
default :
throw new EntityException(
"A package with identifier '{0}' and version '{1}' already exists.",
packageRegistration.Id,
package.Version);
}
}
var now = DateTime.UtcNow;
var packageFileStream = nugetPackage.GetStream();
//if new package versus updating an existing package.
if (package == null) package = new Package();
package.Version = nugetPackage.Version.ToString();
package.Description = nugetPackage.Description;
package.ReleaseNotes = nugetPackage.ReleaseNotes;
package.RequiresLicenseAcceptance = nugetPackage.RequireLicenseAcceptance;
package.HashAlgorithm = Constants.Sha512HashAlgorithmId;
package.Hash = cryptoSvc.GenerateHash(packageFileStream.ReadAllBytes());
package.PackageFileSize = packageFileStream.Length;
package.Created = now;
package.Language = nugetPackage.Language;
package.LastUpdated = now;
package.Published = now;
package.Copyright = nugetPackage.Copyright;
package.IsPrerelease = !nugetPackage.IsReleaseVersion();
package.Listed = false;
package.Status = PackageStatusType.Submitted;
package.SubmittedStatus = PackageSubmittedStatusType.Ready;
package.ApprovedDate = null;
if (package.ReviewedDate.HasValue) package.SubmittedStatus = PackageSubmittedStatusType.Updated;
//we don't moderate prereleases
if (package.IsPrerelease)
{
package.Listed = true;
package.Status = PackageStatusType.Exempted;
}
if (packageRegistration.IsTrusted)
{
package.Listed = true;
package.Status = PackageStatusType.Approved;
package.ReviewedDate = now;
package.ApprovedDate = now;
}
package.IconUrl = nugetPackage.IconUrl == null ? string.Empty : nugetPackage.IconUrl.ToString();
package.LicenseUrl = nugetPackage.LicenseUrl == null ? string.Empty : nugetPackage.LicenseUrl.ToString();
package.ProjectUrl = nugetPackage.ProjectUrl == null ? string.Empty : nugetPackage.ProjectUrl.ToString();
package.ProjectSourceUrl = nugetPackage.ProjectSourceUrl == null
? string.Empty
: nugetPackage.ProjectSourceUrl.ToString();
package.PackageSourceUrl = nugetPackage.PackageSourceUrl == null
? string.Empty
: nugetPackage.PackageSourceUrl.ToString();
package.DocsUrl = nugetPackage.DocsUrl == null ? string.Empty : nugetPackage.DocsUrl.ToString();
package.MailingListUrl = nugetPackage.MailingListUrl == null
? string.Empty
: nugetPackage.MailingListUrl.ToString();
package.BugTrackerUrl = nugetPackage.BugTrackerUrl == null ? string.Empty : nugetPackage.BugTrackerUrl.ToString();
package.Summary = nugetPackage.Summary ?? string.Empty;
package.Tags = nugetPackage.Tags ?? string.Empty;
package.Title = nugetPackage.Title ?? string.Empty;
foreach (var item in package.Authors.OrEmptyListIfNull().ToList())
{
packageAuthorRepo.DeleteOnCommit(item);
}
packageAuthorRepo.CommitChanges();
foreach (var author in nugetPackage.Authors)
{
package.Authors.Add(
new PackageAuthor
{
Name = author
});
}
foreach (var item in package.SupportedFrameworks.OrEmptyListIfNull().ToList())
{
packageFrameworksRepo.DeleteOnCommit(item);
}
packageFrameworksRepo.CommitChanges();
var supportedFrameworks = GetSupportedFrameworks(nugetPackage).Select(fn => fn.ToShortNameOrNull()).ToArray();
if (!supportedFrameworks.AnySafe(sf => sf == null))
{
foreach (var supportedFramework in supportedFrameworks)
{
package.SupportedFrameworks.Add(
new PackageFramework
{
TargetFramework = supportedFramework
});
}
}
foreach (var item in package.Dependencies.OrEmptyListIfNull().ToList())
{
packageDependenciesRepo.DeleteOnCommit(item);
}
packageDependenciesRepo.CommitChanges();
foreach (var dependencySet in nugetPackage.DependencySets)
{
if (dependencySet.Dependencies.Count == 0)
{
package.Dependencies.Add(
new PackageDependency
{
Id = null,
VersionSpec = null,
TargetFramework = dependencySet.TargetFramework.ToShortNameOrNull()
});
} else
{
foreach (var dependency in dependencySet.Dependencies.Select(
d => new
{
d.Id,
d.VersionSpec,
dependencySet.TargetFramework
}))
{
package.Dependencies.Add(
new PackageDependency
{
Id = dependency.Id,
VersionSpec = dependency.VersionSpec == null ? null : dependency.VersionSpec.ToString(),
TargetFramework = dependency.TargetFramework.ToShortNameOrNull()
});
}
}
}
foreach (var item in package.Files.OrEmptyListIfNull().ToList())
{
packageFilesRepo.DeleteOnCommit(item);
}
packageFilesRepo.CommitChanges();
foreach (var packageFile in nugetPackage.GetFiles().OrEmptyListIfNull())
{
var filePath = packageFile.Path;
var fileContent = " ";
IList<string> extensions = new List<string>();
var approvedExtensions = Configuration.ReadAppSettings("PackageFileTextExtensions");
if (!string.IsNullOrWhiteSpace(approvedExtensions))
{
foreach (var extension in approvedExtensions.Split(',', ';'))
{
extensions.Add("." + extension);
}
}
try
{
var extension = Path.GetExtension(filePath);
if (extension != null)
{
if (extensions.Contains(extension)) fileContent = packageFile.GetStream().ReadToEnd();
else if (extension.Equals(".exe", StringComparison.InvariantCultureIgnoreCase))
{
var bytes = packageFile.GetStream().ReadAllBytes();
var md5Hash =
BitConverter.ToString(Convert.FromBase64String(cryptoSvc.GenerateHash(bytes, "MD5")))
.Replace("-", string.Empty);
var sha1Hash =
BitConverter.ToString(Convert.FromBase64String(cryptoSvc.GenerateHash(bytes, "SHA1")))
.Replace("-", string.Empty);
fileContent = string.Format("md5: {0} | sha1: {1}", md5Hash, sha1Hash);
}
}
} catch (Exception ex)
{
// Log but swallow the exception
ErrorSignal.FromCurrentContext().Raise(ex);
}
package.Files.Add(
new PackageFile
{
FilePath = filePath,
FileContent = fileContent,
});
}
package.FlattenedAuthors = package.Authors.Flatten();
package.FlattenedDependencies = package.Dependencies.Flatten();
return package;
}
static List<Package> LoadPackages(JArray asArray)
{
var packages = new List<Package>();
Dictionary<string, User> users = new Dictionary<string, User>();
Func<string, User> getUser = (name) =>
{
if (!users.ContainsKey(name))
{
users[name] = new User { Username = name };
}
return users[name];
};
Func<PseudoDependency, PackageDependency> getDependency = (pd) =>
{
return new PackageDependency
{
TargetFramework = pd.TargetFramework,
VersionSpec = pd.Version,
};
};
Func<PseudoPackageRegistration, PackageRegistration> getPackageRegistration = (pr) =>
{
return new PackageRegistration
{
DownloadCount = pr.DownloadCount,
Id = pr.Id,
Key = pr.Key,
Owners = pr.Owners.Select(name => getUser(name)).ToList(),
};
};
foreach (JObject j in asArray)
{
PseudoPackage pseudo = j.ToObject<PseudoPackage>();
Package p = new Package
{
Authors = pseudo.Authors.Select(name => new PackageAuthor { Name = name }).ToList(),
Copyright = pseudo.Copyright,
Created = pseudo.Created,
Description = pseudo.Description,
Dependencies = pseudo.Dependencies.Select(pd => getDependency(pd)).ToList(),
DownloadCount = pseudo.DownloadCount,
FlattenedAuthors = pseudo.FlattenedAuthors,
FlattenedDependencies = pseudo.FlattenedDependencies,
Hash = pseudo.Hash,
HashAlgorithm = pseudo.HashAlgorithm,
IconUrl = pseudo.IconUrl,
IsLatest = pseudo.IsLatest,
IsLatestStable = pseudo.IsLatestStable,
IsPrerelease = pseudo.IsPrerelease,
Key = pseudo.Key,
Language = pseudo.Language,
LastUpdated = pseudo.LastUpdated,
LicenseUrl = pseudo.LicenseUrl,
Listed = pseudo.Listed,
MinClientVersion = pseudo.MinClientVersion,
PackageFileSize = pseudo.PackageFileSize,
PackageRegistration = getPackageRegistration(pseudo.PackageRegistration),
PackageRegistrationKey = pseudo.PackageRegistrationKey,
ProjectUrl = pseudo.ProjectUrl,
Published = pseudo.Published,
ReleaseNotes = pseudo.ReleaseNotes,
RequiresLicenseAcceptance = pseudo.RequiresLicenseAcceptance,
Summary = pseudo.Summary,
SupportedFrameworks = pseudo.SupportedFrameworks.Select(tf => new PackageFramework { TargetFramework = tf }).ToList(),
Tags = pseudo.Tags,
Title = pseudo.Title,
Version = pseudo.Version,
};
packages.Add(p);
}
return packages;
}
private static void BackgroundJobsPostStart()
{
var jobs = new List<IJob>();
var indexer = DependencyResolver.Current.GetService<IIndexingService>();
if (indexer != null)
{
indexer.RegisterBackgroundJobs(jobs);
}
jobs.Add(new UpdateStatisticsJob(TimeSpan.FromMinutes(5), () => new EntitiesContext(), timeout: TimeSpan.FromMinutes(5)));
jobs.Add(new WorkItemCleanupJob(TimeSpan.FromDays(1), () => new EntitiesContext(), timeout: TimeSpan.FromDays(4)));
var jobCoordinator = new WebFarmJobCoordinator(new EntityWorkItemRepository(() => new EntitiesContext()));
_jobManager = new JobManager(jobs, jobCoordinator)
{
RestartSchedulerOnFailure = true
};
_jobManager.Fail(e => ErrorLog.GetDefault(null).Log(new Error(e)));
_jobManager.Start();
}
public async Task<StatisticsPackagesReport> GetPackageVersionDownloadsByClient(string packageId, string packageVersion)
{
try
{
if (string.IsNullOrEmpty(packageId) || string.IsNullOrEmpty(packageVersion))
{
return null;
}
string reportName = string.Format(CultureInfo.CurrentCulture, "{0}{1}.json", Reports.RecentPopularityDetail_, packageId);
reportName = reportName.ToLowerInvariant();
string json = await _reportService.Load(reportName);
if (json == null)
{
return null;
}
JObject content = JObject.Parse(json);
StatisticsPackagesReport report = new StatisticsPackagesReport();
IList<StatisticsFact> facts = new List<StatisticsFact>();
foreach (StatisticsFact fact in CreateFacts(content))
{
if (fact.Dimensions["Version"] == packageVersion)
{
facts.Add(fact);
}
}
report.Facts = facts;
return report;
}
catch (NullReferenceException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (JsonReaderException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (StorageException e)
{
QuietLog.LogHandledException(e);
return null;
}
catch (ArgumentException e)
{
QuietLog.LogHandledException(e);
return null;
}
}
private static void BackgroundJobsPostStart(IAppConfiguration configuration)
{
var indexer = Container.Kernel.TryGet<IIndexingService>();
var jobs = new List<IJob>();
if (indexer != null)
{
indexer.RegisterBackgroundJobs(jobs, configuration);
}
if (!configuration.HasWorker)
{
jobs.Add(
new UpdateStatisticsJob(TimeSpan.FromMinutes(5),
() => new EntitiesContext(configuration.SqlConnectionString, readOnly: false),
timeout: TimeSpan.FromMinutes(5)));
}
if (configuration.CollectPerfLogs)
{
jobs.Add(CreateLogFlushJob());
}
if (jobs.AnySafe())
{
var jobCoordinator = new NuGetJobCoordinator();
_jobManager = new JobManager(jobs, jobCoordinator)
{
RestartSchedulerOnFailure = true
};
_jobManager.Fail(e => ErrorLog.GetDefault(null).Log(new Error(e)));
_jobManager.Start();
}
}
private static IList<StatisticsFact> CreateFacts(JObject data)
{
IList<StatisticsFact> facts = new List<StatisticsFact>();
foreach (JObject perVersion in data["Items"])
{
string version = (string)perVersion["Version"];
foreach (JObject perClient in perVersion["Items"])
{
string clientName = (string)perClient["ClientName"];
string clientVersion = (string)perClient["ClientVersion"];
string operation = "unknown";
JToken opt;
if (perClient.TryGetValue("Operation", out opt))
{
operation = (string)opt;
}
int downloads = (int)perClient["Downloads"];
facts.Add(new StatisticsFact(CreateDimensions(version, clientName, clientVersion, operation), downloads));
}
}
return facts;
}
