/// <summary>
/// Provides a view of features packages that have recently bee updated in the repository group.
/// Return the three most recently updated packages in the group.
/// </summary>
public ActionResult BrowsePackages()
{
log.DebugFormat("Controller action requested");
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
// Set up the pagination
Pager pager = new Pager(1, SettingsHelper.GetSearchResultsPerPage());
searchParameters.Offset = pager.RecordOffset;
searchParameters.Limit = pager.RecordsPerPage;
PackageSearchResultsModel model = new PackageSearchResultsModel();
model.SearchParameters = searchParameters;
model.SearchResults = CkanHelper.GetClient().SearchPackages <Package>(searchParameters, new CacheSettings(SettingsHelper.GetSearchResultsCacheDuration()));
SettingsHelper.FilterTitles(model.SearchResults.Results);
model.Pager = pager;
// Set the number of records to be paged
pager.RecordCount = model.SearchResults.Count;
// Render the view
return(View(model));
}
public ExplorerPageViewModel(ExplorerPage page, IPackageLoaderService packagesLoaderService,
IModelProvider <ExplorerSettingsContainer> settingsProvider, IPackageMetadataMediaDownloadService packageMetadataMediaDownloadService, INuGetFeedVerificationService nuGetFeedVerificationService,
ICommandManager commandManager, IDispatcherService dispatcherService, IRepositoryContextService repositoryService, ITypeFactory typeFactory,
IDefferedPackageLoaderService defferedPackageLoaderService, IPackageOperationContextService packageOperationContextService, INuGetCacheManager nuGetCacheManager,
INuGetConfigurationService nuGetConfigurationService, IDispatcherProviderService dispatcherProviderService)
{
Argument.IsNotNull(() => packagesLoaderService);
Argument.IsNotNull(() => settingsProvider);
Argument.IsNotNull(() => packageMetadataMediaDownloadService);
Argument.IsNotNull(() => commandManager);
Argument.IsNotNull(() => nuGetFeedVerificationService);
Argument.IsNotNull(() => dispatcherService);
Argument.IsNotNull(() => repositoryService);
Argument.IsNotNull(() => typeFactory);
Argument.IsNotNull(() => defferedPackageLoaderService);
Argument.IsNotNull(() => packageOperationContextService);
Argument.IsNotNull(() => nuGetCacheManager);
Argument.IsNotNull(() => nuGetConfigurationService);
Argument.IsNotNull(() => dispatcherProviderService);
_dispatcherService = dispatcherService;
_packageMetadataMediaDownloadService = packageMetadataMediaDownloadService;
_nuGetFeedVerificationService = nuGetFeedVerificationService;
_repositoryService = repositoryService;
_defferedPackageLoaderService = defferedPackageLoaderService;
_packageOperationContextService = packageOperationContextService;
_typeFactory = typeFactory;
_packagesLoaderService = packagesLoaderService;
_nuGetCacheManager = nuGetCacheManager;
_nuGetConfigurationService = nuGetConfigurationService;
_dispatcherProviderService = dispatcherProviderService;
Settings = settingsProvider.Model;
LoadNextPackagePage = new TaskCommand(LoadNextPackagePageExecuteAsync);
CancelPageLoading = new TaskCommand(CancelPageLoadingExecuteAsync);
RefreshCurrentPage = new TaskCommand(RefreshCurrentPageExecuteAsync);
commandManager.RegisterCommand(nameof(RefreshCurrentPage), RefreshCurrentPage, this);
Title = page.Parameters.Tab.Name;
_initialSearchParams = page.Parameters.InitialSearchParameters; //if null, standard Settings will not be overriden
if (Title != "Browse")
{
#pragma warning disable IDISP004 // Don't ignore created IDisposable.
_packagesLoaderService = this.GetServiceLocator().ResolveType <IPackageLoaderService>(Title);
#pragma warning restore IDISP004 // Don't ignore created IDisposable.
}
if (!Enum.TryParse(Title, out _pageType))
{
Log.Error("Unrecognized page type");
}
CanBatchProjectActions = _pageType != MetadataOrigin.Installed;
Page = page;
}
/// <summary>
/// Get a count of all packages that meet the search criteria with caching.
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <returns>Search results</returns>
public int GetPackageCount(PackageSearchParameters parameters, CacheSettings settings)
{
// Set the limit to 0 (only getting the count, not package details)
parameters.Limit = 0;
int count = SearchPackages <string>(parameters, settings).Count;
return(count);
}
/// <summary>
/// Get the number of packages in the CKAN group.
/// </summary>
/// <returns></returns>
public static int GetPackageCount()
{
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
searchParameters.Limit = 0;
int count = CkanHelper.GetClient().SearchPackages <string>(searchParameters, new CacheSettings(SettingsHelper.GetPackageCountCacheDuration(), SettingsHelper.GetPackageCountCacheBackgroundUpdate())).Count;
return(count);
}
public void ShouldReturnPackageIdsWithQuery()
{
CkanClient client = CkanApiHelper.GetCkanClient();
PackageSearchParameters parameters = new PackageSearchParameters();
parameters.Query = "bike";
PackageSearchResponse <string> response = client.SearchPackages <string>(parameters);
Assert.NotEmpty(response.Results);
}
public void ShouldReturnPackagesWithTag()
{
CkanClient client = CkanApiHelper.GetCkanClient();
PackageSearchParameters parameters = new PackageSearchParameters();
parameters.Tags.Add("colorado");
PackageSearchResponse <Package> response = client.SearchPackages <Package>(parameters);
Assert.NotEmpty(response.Results);
}
/// <summary>
/// Search for packages in the CKAN repository and just return tag counts with caching.
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <param name="settings">The cache settings.</param>
/// <returns>Search results</returns>
public List <Tag> SearchPackagesGetTagCounts(PackageSearchParameters parameters, CacheSettings settings)
{
// Execute the search request
parameters.Offset = 0;
parameters.Limit = GetPackageCount(parameters, settings);
PackageSearchResponse <Package> response = SearchPackages <Package>(parameters, settings);
List <Package> packages = response.Results;
List <Tag> tags = TagHelper.GetTagCounts(packages);
return(tags);
}
public void ShouldReturnSinglePackagesWithQueryLimit()
{
CkanClient client = CkanApiHelper.GetCkanClient();
PackageSearchParameters parameters = new PackageSearchParameters();
parameters.Query = "bike";
parameters.Offset = 0;
parameters.Limit = 1;
PackageSearchResponse <Package> response = client.SearchPackages <Package>(parameters);
Assert.True(response.Results.Count == 1);
}
/// <summary>
/// Get all packages from the CKAN group.
/// </summary>
/// <returns></returns>
public static List <Package> GetAllPackages()
{
CacheSettings settings = new CacheSettings(SettingsHelper.GetAllPackagesCacheDuration(), SettingsHelper.GetAllPackagesCacheBackgroundUpdate());
// Create the CKAN search parameters
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
searchParameters.Limit = GetPackageCount();
// Collect the results
PackageSearchResponse <Package> response = CkanHelper.GetClient().SearchPackages <Package>(searchParameters, settings);
return(response.Results);
}
/// <summary>
/// Get search suggestions for all packages in the catalog group. Uses the all packages cache
/// to build a search index that is also cached for performance.
/// </summary>
/// <param name="prefix"></param>
/// <returns></returns>
public static List <string> GetPackageSearchSuggestions(string prefix)
{
CacheSettings settings = new CacheSettings(SettingsHelper.GetAllPackagesCacheDuration(), SettingsHelper.GetAllPackagesCacheBackgroundUpdate());
// Create the CKAN search parameters to get all packages in the group
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
searchParameters.Limit = GetPackageCount();
// Get a list of search suggestions
List <string> suggestions = CkanHelper.GetClient().GetPackageSearchSuggestions(prefix, searchParameters, settings);
return(suggestions);
}
public void ShouldReturnPackagesWithGroup()
{
CkanClient client = CkanApiHelper.GetCkanClient();
PackageSearchParameters parameters = new PackageSearchParameters();
parameters.Groups.Add("arvada");
PackageSearchResponse <Package> response = client.SearchPackages <Package>(parameters);
Assert.NotEmpty(response.Results);
foreach (var result in response.Results)
{
Assert.Contains <string>("arvada", result.Groups);
}
}
protected override async Task InitializeAsync()
{
try
{
if (IsActive && _initialSearchParams is not null)
{
// Set page initial search params as Settings parameters
// Only on first loaded page
Settings.IsPreReleaseIncluded = _initialSearchParams.IsPrereleaseIncluded;
Settings.SearchString = _initialSearchParams.SearchString;
Settings.IsRecommendedOnly = _initialSearchParams.IsRecommendedOnly;
}
// Execution delay
SingleDelayTimer.Elapsed += OnTimerElapsed;
SingleDelayTimer.AutoReset = false;
SingleDelayTimer.SynchronizingObject = _typeFactory.CreateInstanceWithParameters <ISynchronizeInvoke>(
_dispatcherProviderService.GetCurrentDispatcher());
PackageItems = new FastObservableCollection <NuGetPackage>();
PackageItems.CollectionChanged += OnPackageItemsCollectionChanged;
_packageOperationContextService.OperationContextDisposing += OnOperationContextDisposing;
IsFirstLoaded = false;
var pageSize = _nuGetConfigurationService.GetPackageQuerySize();
if (Settings.ObservedFeed is not null && !string.IsNullOrEmpty(Settings.ObservedFeed.Source))
{
var currentFeed = Settings.ObservedFeed;
PageInfo = new PageContinuation(pageSize, Settings.ObservedFeed.GetPackageSource());
var searchParams = new PackageSearchParameters(Settings.IsPreReleaseIncluded, Settings.SearchString, Settings.IsRecommendedOnly);
await VerifySourceAndLoadPackagesAsync(PageInfo, currentFeed, searchParams, pageSize);
}
else
{
Log.Info("None of the source feeds configured");
}
}
/// <summary>
/// Provides a view of features packages in the repository group. Featured
/// packages are those that are tagged with "featured".
/// </summary>
public ActionResult FeaturedPackages()
{
log.DebugFormat("Controller action requested");
// Create the CKAN search parameters
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
searchParameters.Limit = SettingsHelper.GetHomeFeaturedPackageLimit();
searchParameters.Tags.Add(SettingsHelper.GetHomeFeaturedPackagesTag());
// Collect the results
PackageSearchResponse <Package> results = CkanHelper.GetClient().SearchPackages <Package>(searchParameters, new CacheSettings(SettingsHelper.GetFeaturedPackagesCacheDuration()));
SettingsHelper.FilterTitles(results.Results);
// Render the view
return(View(results));
}
/// <summary>
/// Get all tags from the CKAN group.
/// </summary>
/// <returns></returns>
public static List <Tag> GetAllTags(int limit)
{
CacheSettings settings = new CacheSettings(SettingsHelper.GetAllPackagesCacheDuration(), SettingsHelper.GetAllPackagesCacheBackgroundUpdate());
// Create the CKAN search parameters
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
// Collect the results
List <Tag> tags = CkanHelper.GetClient().SearchPackagesGetTagCounts(searchParameters, settings);
// Remove filtered tags
tags = TagHelper.FilterTags(tags);
if (limit > 0)
{
tags = TagHelper.LimitTags(tags, limit);
}
return(tags);
}
/// <summary>
/// Gets a trie of package search suggestions a package search.
/// If cached, the trie will be returned from the
/// cache, otherwise it will be created and cached.
/// </summary>
/// <param name="client"></param>
/// <param name="searchParameters"></param>
/// <param name="cacheSettings"></param>
/// <returns></returns>
public static Trie GetTrie(CkanClient client, PackageSearchParameters searchParameters, CacheSettings cacheSettings)
{
Trie trie = null;
// Get the trie from the cache
string key = CkanClient.CacheKeyPrefix + "AutoComplete";
// Get the memorycache
MemoryCache cache = MemoryCache.Default;
// Get the cached entry if it exists
CacheEntry cacheEntry = cache[key] as CacheEntry;
if (cacheEntry == null)
{
// Generate the trie
trie = GenerateTrie(client, searchParameters, cacheSettings);
cacheEntry = new CacheEntry();
cacheEntry.Data = trie;
cacheEntry.Label = "Autocomplete index";
cacheEntry.LastCached = DateTime.Now;
cacheEntry.Duration = cacheSettings.Duration;
CacheItemPolicy policy = new CacheItemPolicy();
policy.AbsoluteExpiration = DateTimeOffset.Now.Add(cacheSettings.Duration);
// Add the trie to the cache
cache.Set(key, cacheEntry, policy);
}
else
{
// Get the trie from the cache
trie = (Trie)cacheEntry.Data;
}
return(trie);
}
/// <summary>
/// Generates a trie of suggestions based on attributes of packages in the catalog
/// </summary>
/// <param name="client"></param>
/// <param name="searchParameters"></param>
/// <param name="cacheSettings"></param>
/// <returns></returns>
private static Trie GenerateTrie(CkanClient client, PackageSearchParameters searchParameters, CacheSettings cacheSettings)
{
// Create an empty trie
Trie trie = new Trie();
// Run the search to get all packages
PackageSearchResponse <Package> response = client.SearchPackages <Package>(searchParameters, cacheSettings);
// Add the entrys to the trie
foreach (var package in response.Results)
{
// Add the package title
trie.Add(package.Title);
// Add the package tags (removing hyphens which represent spaces)
foreach (var tag in package.Tags)
{
trie.Add(tag.Replace("-", " "));
}
}
return(trie);
}
/// <summary>
/// Provides a view of features packages that have recently bee updated in the repository group.
/// Return the three most recently updated packages in the group.
/// </summary>
public ActionResult RecentlyUpdated()
{
log.DebugFormat("Controller action requested");
// Create the CKAN search parameters
var searchParameters = new PackageSearchParameters();
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
// Collect the results
List <Package> packages = CkanHelper.GetAllPackages();
// Sort by date and take the top 3
packages = (from package in packages
orderby package.MetadataModified
descending
select package)
.Take(3)
.ToList();
SettingsHelper.FilterTitles(packages);
// Update the results to the top 3
PackageSearchResponse <Package> response = new PackageSearchResponse <Package>();
response.Results = packages;
response.Count = packages.Count;
// Populate the model
PackageSearchResultsModel model = new PackageSearchResultsModel();
model.SearchResults = response;
// Render the view
return(View(model));
}
public ActionResult Index(string q, int?page, string order_by, string tag, string mode)
{
log.DebugFormat("Controller action requested");
this.ConfigureBreadCrumbs(q, tag);
// Create the CKAN search parameters
var searchParameters = new PackageSearchParameters();
searchParameters.AggregateTagCounts = true;
searchParameters.Query = q;
searchParameters.Groups.Add(SettingsHelper.GetCatalogGroup());
// Ordering
if (!String.IsNullOrEmpty(order_by))
{
searchParameters.OrderBy = order_by;
}
// Tag
if (!String.IsNullOrEmpty(tag))
{
searchParameters.Tags.Add(tag);
}
// Build the view model for the results
PackageSearchResultsModel model = new PackageSearchResultsModel();
Pager pager = null;
if (mode == "table")
{
model.DisplayMode = ResultsDisplayMode.Table;
searchParameters.Offset = 0;
// TODO: In the table mode all results are currently returned and paginated client side.
// This isn't scalable so this will need looked at in the future (AJAX pagination maybe?)
searchParameters.Limit = CkanHelper.GetPackageCount();
}
else // mode == list (default)
{
// Set up the pagination
int pageNumber = 1;
if (page != null)
{
pageNumber = (int)page;
}
pager = new Pager(pageNumber, SettingsHelper.GetSearchResultsPerPage());
searchParameters.Offset = pager.RecordOffset;
searchParameters.Limit = pager.RecordsPerPage;
}
// Execute the search
model.SearchParameters = searchParameters;
model.SearchResults = CkanHelper.GetClient().SearchPackages <Package>(searchParameters, new CacheSettings(SettingsHelper.GetSearchResultsCacheDuration()));
// Filter the titles
SettingsHelper.FilterTitles(model.SearchResults.Results);
// Filter the tags
model.SearchResults.Tags = TagHelper.FilterTags(model.SearchResults.Tags);
// Set the pager if we are using it
if (pager != null)
{
// Set the number of records to be paged
pager.RecordCount = model.SearchResults.Count;
// Add the pager to the model
model.Pager = pager;
}
// Render the view
return(View(model));
}
/// <summary>
/// Get the top package search suggestions for the data catalog group.
/// </summary>
/// <param name="prefix"></param>
/// <param name="suggestionCount"></param>
/// <param name="searchParameters"></param>
/// <param name="cacheSettings"></param>
/// <returns></returns>
public List <string> GetPackageSearchSuggestions(string prefix, int suggestionCount, PackageSearchParameters searchParameters, CacheSettings cacheSettings)
{
var list = new List <string>();
var trie = SuggestionHelper.GetTrie(this, searchParameters, cacheSettings);
if (trie != null)
{
list = trie.GetCompletionList(prefix);
list.Sort();
list = list.Take(suggestionCount).ToList();
}
return(list);
}
/// <summary>
/// Get the top 10 package search suggestions for the data catalog group.
/// </summary>
/// <param name="prefix"></param>
/// <param name="searchParameters"></param>
/// <param name="cacheSettings"></param>
/// <returns></returns>
public List <string> GetPackageSearchSuggestions(string prefix, PackageSearchParameters searchParameters, CacheSettings cacheSettings)
{
return(GetPackageSearchSuggestions(prefix, 10, searchParameters, cacheSettings));
}
/// <summary>
/// Get a count of all packages that meet the search criteria
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <returns>Search results</returns>
public int GetPackageCount(PackageSearchParameters parameters)
{
return(GetPackageCount(parameters, null));
}
/// <summary>
/// Search for packages in the CKAN repository and just return tag counts.
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <returns>Search results</returns>
private List <Tag> SearchPackagesGetTagCounts(PackageSearchParameters parameters)
{
return(SearchPackagesGetTagCounts(parameters, null));
}
/// <summary>
/// Search for packages in the CKAN repository.
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <returns>Search results</returns>
public PackageSearchResponse <T> SearchPackages <T>(PackageSearchParameters parameters)
{
return(SearchPackages <T>(parameters, null));
}
/// <summary>
/// Search for packages in the CKAN repository with caching.
/// </summary>
/// <param name="parameters">Provides that parameters to use in the search.</param>
/// <param name="settings">The cache settings.</param>
/// <returns>Search results</returns>
public PackageSearchResponse <T> SearchPackages <T>(PackageSearchParameters parameters, CacheSettings settings)
{
var request = new RestRequest();
request.Resource = "2/search/package";
// Apply query parameter
foreach (var group in parameters.Groups)
{
request.AddParameter("q", "organization:" + group);
}
// Apply title parameter
if (!String.IsNullOrEmpty(parameters.Title))
{
request.AddParameter("title", parameters.Title);
}
// Apply tag parameters
foreach (var tag in parameters.Tags)
{
request.AddParameter("tags", tag);
}
// Apply notes parameter
if (!String.IsNullOrEmpty(parameters.Notes))
{
request.AddParameter("notes", parameters.Notes);
}
//// Apply group parameters
//foreach (var group in parameters.Groups)
//{
// request.AddParameter("groups", group);
//}
// Apply author parameter
if (!String.IsNullOrEmpty(parameters.Author))
{
request.AddParameter("author", parameters.Author);
}
// Apply update_frequency parameter
if (!String.IsNullOrEmpty(parameters.UpdateFrequency))
{
request.AddParameter("update_frequency", parameters.UpdateFrequency);
}
// Apply maintainer parameter
if (!String.IsNullOrEmpty(parameters.Maintainer))
{
request.AddParameter("maintainer", parameters.Maintainer);
}
// Apply all_fields parameter
if (typeof(T) == typeof(Package))
{
request.AddParameter("all_fields", 1);
}
// Apply filter_by_openness parameter
if (parameters.FilterByOpenness)
{
request.AddParameter("filter_by_openness", "1");
}
// Apply filter_by_downloadable parameter
if (parameters.FilterByDownloadable)
{
request.AddParameter("filter_by_downloadable", "1");
}
// Apply order_by parameter
if (!String.IsNullOrEmpty(parameters.OrderBy))
{
request.AddParameter("order_by", parameters.OrderBy);
}
// Set the offset and limit parameters
if (parameters.Offset > -1)
{
request.AddParameter("offset", parameters.Offset);
}
if (parameters.Limit > -1)
{
request.AddParameter("limit", parameters.Limit);
}
// Execute the search request
PackageSearchResponse <T> response = Execute <PackageSearchResponse <T> >(request, settings);
// If no results, return an empty results object
if (response == null)
{
response = new PackageSearchResponse <T>();
}
else
{
if (typeof(T) == typeof(Package) && parameters.AggregateTagCounts && response.Results.Count > 0)
{
if (response.Results.Count == response.Count)
{
// If we have all of the results on the current page just aggregate the tags
if (typeof(T) == typeof(Package))
{
List <Package> packages = response.Results.ConvertAll(item => (Package)((object)item));
response.Tags = TagHelper.GetTagCounts(packages);
}
}
else
{
// If we don't have all of the results submit another request to
// get all of the tags
response.Tags = SearchPackagesGetTagCounts(parameters, settings);
}
}
}
return(response);
}