/// <summary>
/// Initializes a new instance of the <see cref="T:Terradue.Tep.WpsJobOpenSearchable"/> class.
/// </summary>
/// <param name="url">URL.</param>
/// <param name="ose">Ose.</param>
public WpsJobProductOpenSearchable(WpsJob job, IfyContext context)
{
this.context = context;
this.Wpsjob = job;
this.openSearchable = job.GetProductOpenSearchable();
}
public static Uri SubstituteUrlApi(Uri url, IOpenSearchable openSearchable, IOpenSearchResultItem item)
{
UriBuilder urib = new UriBuilder(url);
urib.Path = "";
urib.Query = null;
urib.Fragment = null;
var substUri = FindSubstituteUri(urib.Uri, true);
if (substUri == null)
{
return(null);
}
urib = new UriBuilder(substUri);
if (openSearchable is SandboxOpenSearchable)
{
urib.Path += RewritePath(url.AbsolutePath, openSearchable);
}
else
{
urib.Path += RewriteExternalPath(url);
}
return(urib.Uri);
}
public static SyndicationLink SubstituteEnclosure(SyndicationLink link, IOpenSearchable os, IOpenSearchResultItem item)
{
if (os is SandboxOpenSearchable)
{
return(SubstituteSandboxEnclosure(link, os as SandboxOpenSearchable, item));
}
return(SubstituteExternalEnclosure(link));
}
private IOpenSearchResultItem FindReferenceItem(IOpenSearchResultItem item, IOpenSearchable os)
{
FiltersDefinition filters = new FiltersDefinition(item.Identifier);
filters.AddFilter("uid", "{http://a9.com/-/opensearch/extensions/geo/1.0/}uid", item.Identifier, item.Identifier, null, null);
var result = ose.Query(os, filters.GetNameValueCollection());
return(result.Items.FirstOrDefault());
}
private NameValueCollection PrepareQueryParameters(IOpenSearchable entity)
{
NameValueCollection nvc = new NameValueCollection();
totalResults = 0;
foreach (string parameter in parameterArgs)
{
Match matchParamDef = Regex.Match(parameter, @"^(.*)=(.*)$");
// if martch is successful
if (matchParamDef.Success)
{
// TODO filter and convert query param
if (matchParamDef.Groups[1].Value == "count")
{
if (matchParamDef.Groups[2].Value == "unlimited")
{
nvc.Add(matchParamDef.Groups[1].Value, pagination.ToString());
totalResults = int.MaxValue;
continue;
}
totalResults = int.Parse(matchParamDef.Groups[2].Value);
if (totalResults > pagination)
{
nvc.Add(matchParamDef.Groups[1].Value, pagination.ToString());
continue;
}
}
nvc.Add(matchParamDef.Groups[1].Value, matchParamDef.Groups[2].Value);
}
}
if (totalResults == 0)
{
totalResults = pagination;
nvc.Add("count", pagination.ToString());
}
NameValueCollection remoteParams = entity.GetOpenSearchParameters(entity.DefaultMimeType);
if (remoteParams["do"] != null && nvc["do"] == null)
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOWNLOAD_ORIGIN")))
{
nvc["do"] = Environment.GetEnvironmentVariable("DOWNLOAD_ORIGIN");
}
else
{
nvc["do"] = Dns.GetHostName();
}
}
dataModel.SetQueryParameters(nvc);
return(nvc);
}
public static void ProxyOpenSearchResult(IOpenSearchable entity, OpenSearchRequest request, IOpenSearchResultCollection osr)
{
if (!(entity is IProxiedOpenSearchable))
{
return;
}
OpenSearchFactory.ReplaceSelfLinks(entity, request, osr, Terradue.Metadata.EarthObservation.OpenSearch.Helpers.OpenSearchParametersHelper.EntrySelfLinkTemplate);
OpenSearchFactory.ReplaceOpenSearchDescriptionLinks(entity, osr);
}
/// <summary>
/// Initializes a new instance of the <see cref="Terradue.OpenSearch.Request.MultiAtomOpenSearchRequest"/> class.
/// </summary>
/// <param name="ose">Instance of OpenSearchEngine, preferably with the AtomOpenSearchEngineExtension registered</param>
/// <param name="entities">IOpenSearchable entities to be searched.</param>
/// <param name="type">contentType of the .</param>
/// <param name="url">URL.</param>
public MultiAtomGroupedOpenSearchRequest(OpenSearchEngine ose, IOpenSearchable[] entities, string type, OpenSearchUrl url, bool concurrent)
: base(url, type)
{
this.concurrent = concurrent;
this.ose = ose;
this.type = type;
this.parameters = HttpUtility.ParseQueryString(url.Query);
this.entities = entities;
}
private IOpenSearchResultCollection QueryOpenSearch(OpenSearchEngine ose, IOpenSearchable entity, NameValueCollection parameters)
{
IOpenSearchResultCollection osr;
if (string.IsNullOrEmpty(queryFormatArg))
{
osr = ose.Query(entity, parameters);
}
else
{
osr = ose.Query(entity, parameters, queryFormatArg);
}
return(osr);
}
public static string RewritePath(string path, IOpenSearchable openSearchable)
{
if (openSearchable is SandboxOpenSearchable)
{
var sandboxOpenSearchable = openSearchable as SandboxOpenSearchable;
var match = Regex.Match(path, string.Format(".*\\/{0}\\/_results\\/(?'relativeFilename'.*)", sandboxOpenSearchable.SandboxOpenSearchInformation.RunId));
if (match.Success)
{
return(path = string.Format("/production/workflows/{0}/runs/{1}/{2}", sandboxOpenSearchable.SandboxOpenSearchInformation.Workflow,
sandboxOpenSearchable.SandboxOpenSearchInformation.RunId,
match.Groups["relativeFilename"].Value));
}
}
return(path);
}
public static void ReplaceSelfLinks(IOpenSearchable entity, NameValueCollection parameters, IOpenSearchResultCollection osr, Func<IOpenSearchResultItem,OpenSearchDescription,string,string> entryTemplate, string contentType)
{
IOpenSearchResultCollection feed = osr;
var matchLinks = feed.Links.Where(l => l.RelationshipType == "self").ToArray();
foreach (var link in matchLinks) {
feed.Links.Remove(link);
}
OpenSearchDescription osd = null;
if (entity is IProxiedOpenSearchable) {
osd = ((IProxiedOpenSearchable)entity).GetProxyOpenSearchDescription();
} else {
osd = entity.GetOpenSearchDescription();
}
if (OpenSearchFactory.GetOpenSearchUrlByType(osd, contentType) == null)
return;
NameValueCollection newNvc = new NameValueCollection(parameters);
NameValueCollection osparams = OpenSearchFactory.GetOpenSearchParameters(OpenSearchFactory.GetOpenSearchUrlByType(osd, contentType));
newNvc.AllKeys.FirstOrDefault(k => {
if (string.IsNullOrEmpty(OpenSearchFactory.GetParamNameFromId(osparams, k)))
newNvc.Remove(k);
return false;
});
osparams.AllKeys.FirstOrDefault(k => {
Match matchParamDef = Regex.Match(osparams[k], @"^{([^?]+)\??}$");
if (!matchParamDef.Success)
newNvc.Set(k, osparams[k]);
return false;
});
UriBuilder myUrl = new UriBuilder(OpenSearchFactory.GetOpenSearchUrlByType(osd, contentType).Template);
string[] queryString = Array.ConvertAll(newNvc.AllKeys, key => string.Format("{0}={1}", key, newNvc[key]));
myUrl.Query = string.Join("&", queryString);
feed.Links.Add(new SyndicationLink(myUrl.Uri, "self", "Reference link", contentType, 0));
feed.Id = myUrl.ToString();
foreach (IOpenSearchResultItem item in feed.Items) {
matchLinks = item.Links.Where(l => l.RelationshipType == "self").ToArray();
foreach (var link in matchLinks) {
item.Links.Remove(link);
}
string template = entryTemplate(item, osd, contentType);
if (template != null) {
item.Links.Add(new SyndicationLink(new Uri(template), "self", "Reference link", contentType, 0));
item.Id = template;
}
}
}
public static void ReplaceOpenSearchDescriptionLinks(IOpenSearchable entity, IOpenSearchResultCollection osr)
{
IOpenSearchResultCollection feed = osr;
var matchLinks = feed.Links.Where(l => l.RelationshipType == "search").ToArray();
foreach (var link in matchLinks) {
feed.Links.Remove(link);
}
OpenSearchDescription osd;
if (entity is IProxiedOpenSearchable)
osd = ((IProxiedOpenSearchable)entity).GetProxyOpenSearchDescription();
else
osd = entity.GetOpenSearchDescription();
OpenSearchDescriptionUrl url = OpenSearchFactory.GetOpenSearchUrlByRel(osd, "self");
Uri uri;
if (url != null)
uri = new Uri(url.Template);
else
uri = osd.Originator;
if (uri != null)
feed.Links.Add(new SyndicationLink(uri, "search", "OpenSearch Description link", "application/opensearchdescription+xml", 0));
}
public static void ReplaceSelfLinks(IOpenSearchable entity, NameValueCollection parameters, IOpenSearchResultCollection osr, Func<IOpenSearchResultItem,OpenSearchDescription,string,string> entryTemplate)
{
ReplaceSelfLinks(entity, parameters, osr, entryTemplate, osr.ContentType);
}
/// <summary>
/// Gets the name of the identifier from parameter.
/// </summary>
/// <returns>The identifier from parameter name.</returns>
/// <param name="entity">Entity.</param>
/// <param name="mimeType">MIME type.</param>
/// <param name="paramName">Parameter name.</param>
public static string GetIdFromParamName(IOpenSearchable entity, string mimeType, string paramName)
{
NameValueCollection nvc = entity.GetOpenSearchParameters(mimeType);
NameValueCollection revNvc = ReverseTemplateOpenSearchParameters(nvc);
return revNvc[paramName];
}
/// <summary>
/// Merges the open search parameters.
/// </summary>
/// <returns>The open search parameters.</returns>
/// <param name="entities">Entities.</param>
/// <param name="contentType">Content type.</param>
public static NameValueCollection MergeOpenSearchParameters(IOpenSearchable[] entities, string contentType)
{
NameValueCollection nvc = new NameValueCollection();
foreach (IOpenSearchable entity in entities) {
NameValueCollection nvc2 = entity.GetOpenSearchParameters(contentType);
int i = 1;
foreach (string param in nvc2.Keys) {
if (nvc2[param] == nvc[param])
continue;
if (nvc[param] == null) {
nvc.Add(param, nvc2[param]);
continue;
}
if (nvc[param] != null) {
nvc.Add(param + i++, nvc2[param]);
continue;
}
}
}
return nvc;
}
private IOpenSearchResultCollection QueryOpenSearch(OpenSearchEngine ose, IOpenSearchable entity, NameValueCollection parameters)
{
IOpenSearchResultCollection osr;
if (string.IsNullOrEmpty(queryFormatArg))
osr = ose.Query(entity, parameters);
else
osr = ose.Query(entity, parameters, queryFormatArg);
return osr;
}
public abstract void AddSearchLink(ref IOpenSearchResultCollection osr, NameValueCollection originalParameters, IOpenSearchable entity, string with, string finalContentType);
private bool ProcessAlternative(List <Uri> uri, List <NetworkCredential> credential, ref bool isAtomFeedPartial, ref bool canceled)
{
// Find OpenSearch Entity
IOpenSearchable entity = null;
int retry = retryAttempts;
int index = 1;
while (retry >= 0)
{
// Perform the query
try {
OpenSearchableFactorySettings settings = new OpenSearchableFactorySettings(ose);
settings.MaxRetries = retryAttempts;
entity = dataModel.CreateOpenSearchable(uri, queryFormatArg, ose, credential, settings);
index = entity.GetOpenSearchDescription().DefaultUrl.IndexOffset;
log.Debug("IndexOffset : " + index);
break;
} catch (Exception e) {
log.Warn(e.Message);
if (retry == 0)
{
throw;
}
retry--;
searchCache.ClearCache(".*", DateTime.Now);
}
}
NameValueCollection parameters = PrepareQueryParameters(entity);
string startIndex = parameters.Get("startIndex");
if (startIndex != null)
{
index = int.Parse(startIndex);
}
NameValueCollection parametersNvc;
IOpenSearchResultCollection osr = null;
long totalCount = 0;
log.DebugFormat("{0} entries requested", totalResults);
if (outputStarted)
{
return(false);
}
while (totalResults > 0)
{
bool closeOutputStream = true;
Stream outputStream = null;
log.DebugFormat("startIndex: {0}", index);
parametersNvc = ResolveParameters(parameters, entity);
// Initialize the output stream
if (outputStream == null)
{
outputStream = InitializeOutputStream(index);
}
else
{
closeOutputStream = false;
}
retry = retryAttempts;
while (retry >= 0)
{
if (canceled)
{
return(false);
}
// Perform the query
log.Debug("Launching query...");
try {
osr = QueryOpenSearch(ose, entity, parametersNvc);
isAtomFeedPartial = false;
break;
} catch (AggregateException ae) {
if (retry == 0)
{
throw ae;
}
foreach (Exception e in ae.InnerExceptions)
{
log.Warn("Exception " + e.Message);
}
retry--;
searchCache.ClearCache(".*", DateTime.Now);
} catch (KeyNotFoundException e) {
log.Error("Query not found : " + e.Message);
throw e;
} catch (PartialAtomException e) {
if (retry == 0)
{
osr = e.PartialOpenSearchResultCollection;
isAtomFeedPartial = true;
}
retry--;
searchCache.ClearCache(".*", DateTime.Now);
} catch (ThreadAbortException) {
} catch (Exception e) {
if (retry == 0)
{
throw;
}
log.Warn("Exception " + e.Message);
retry--;
searchCache.ClearCache(".*", DateTime.Now);
}
}
if (canceled)
{
return(false);
}
int count = CountResults(osr);
if (totalCount == 0 && count == 0)
{
LogInfo("No entries found");
DeleteFileStream(outputStream);
return(false);
}
if (canceled)
{
return(false);
}
if (osr.Count > 0)
{
// Transform the result
OutputResult(osr, outputStream);
}
else
{
closeOutputStream = false;
DeleteFileStream(outputStream);
}
outputStarted = true;
log.Debug(count + " entries found");
if (count == 0)
{
break;
}
int expectedCount = count;
if (!string.IsNullOrEmpty(parameters["count"]) && int.TryParse(parameters["count"], out expectedCount) && count < expectedCount)
{
break;
}
totalResults -= count;
totalCount += count;
log.Debug(count + " entries found on " + totalResults + " requested");
int paramCount;
if (Int32.TryParse(parameters.Get("count"), out paramCount) && totalResults < paramCount)
{
parameters.Set("count", "" + totalResults);
}
index += count;
parameters.Set("startIndex", "" + index);
if (closeOutputStream)
{
outputStream.Close();
}
}
return(totalCount > 0); // success
}
private NameValueCollection ResolveParameters(NameValueCollection nameValueCollection, IOpenSearchable entity)
{
string contentType = entity.DefaultMimeType;
NameValueCollection osdParam = entity.GetOpenSearchParameters(contentType);
NameValueCollection osdRevParams = OpenSearchFactory.ReverseTemplateOpenSearchParameters(osdParam);
NameValueCollection parameters = new NameValueCollection();
foreach (string key in nameValueCollection.AllKeys)
{
if (osdRevParams[key] != null)
{
foreach (string id in osdRevParams.GetValues(key))
{
parameters.Set(id, nameValueCollection[key]);
}
}
else
{
parameters.Set(key, nameValueCollection[key]);
}
}
return(parameters);
}
internal TestUnitResult MakeQuery(IOpenSearchable entity, FiltersDefinition fd)
{
List <IMetric> metrics = new List <IMetric>();
log.DebugFormat("[{1}] > Query {0} {2}...", fd.Name, Task.CurrentId, fd.Label);
Stopwatch sw = new Stopwatch();
sw.Start();
DateTimeOffset timeStart = DateTimeOffset.UtcNow;
var parameters = fd.GetNameValueCollection();
return(catalogue_task_factory.StartNew(() =>
{
return ose.Query(entity, parameters);
}).ContinueWith <TestUnitResult>(task =>
{
var respTime = sw.ElapsedMilliseconds;
Terradue.OpenSearch.Result.IOpenSearchResultCollection results = null;
try
{
results = task.Result;
}
catch (AggregateException e)
{
log.DebugFormat("[{0}] < No results for {2}. Exception: {1}", Task.CurrentId, e.InnerException.Message, fd.Label);
log.Debug(e.InnerException.StackTrace);
metrics.Add(new ExceptionMetric(e.InnerException));
metrics.Add(new LongMetric(MetricName.maxTotalResults, -1, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, -1, "#"));
}
finally
{
sw.Stop();
}
DateTimeOffset timeStop = DateTimeOffset.UtcNow;
metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime"));
metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime"));
metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string"));
if (results != null)
{
foundItems.AddRange(results.Items);
long serializedSize = 0;
try
{
serializedSize = Encoding.Default.GetBytes(results.SerializeToString()).Length;
}
catch
{
}
var metricsArray = results.ElementExtensions.ReadElementExtensions <Terradue.OpenSearch.Benchmarking.Metrics>("Metrics", "http://www.terradue.com/metrics", Terradue.OpenSearch.Benchmarking.MetricFactory.Serializer);
if (metricsArray == null || metricsArray.Count() == 0)
{
log.Warn("No query metrics found! Response Time and error rate may be biased!");
metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms"));
metrics.Add(new LongMetric(MetricName.size, serializedSize, "bytes"));
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, timeStart.Ticks, "ticks"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, timeStop.Ticks, "ticks"));
}
else
{
Terradue.OpenSearch.Benchmarking.Metrics osMetrics = metricsArray.First();
Terradue.OpenSearch.Benchmarking.Metric _sizeMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "size");
Terradue.OpenSearch.Benchmarking.Metric _responseTimeMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "responseTime");
Terradue.OpenSearch.Benchmarking.Metric _retryNumberMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "retryNumber");
Terradue.OpenSearch.Benchmarking.Metric _beginGetResponseTime = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "beginGetResponseTime");
Terradue.OpenSearch.Benchmarking.Metric _endGetResponseTime = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "endGetResponseTime");
log.DebugFormat("[{4}] < {0}/{1} entries for {6} {5}. {2}bytes in {3}ms", results.Count, results.TotalResults, _sizeMetric.Value, _responseTimeMetric.Value, Task.CurrentId, fd.Label, fd.Name);
if (_responseTimeMetric != null)
{
metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(_responseTimeMetric.Value), "ms"));
}
else
{
metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms"));
}
if (_sizeMetric != null)
{
metrics.Add(new LongMetric(MetricName.size, Convert.ToInt64(_sizeMetric.Value), "bytes"));
}
else
{
metrics.Add(new LongMetric(MetricName.size, serializedSize, "bytes"));
}
if (_retryNumberMetric != null)
{
metrics.Add(new LongMetric(MetricName.retryNumber, Convert.ToInt64(_retryNumberMetric.Value), "#"));
}
else
{
metrics.Add(new LongMetric(MetricName.retryNumber, 1, "#"));
}
if (_beginGetResponseTime != null && _endGetResponseTime != null)
{
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, Convert.ToInt64(_beginGetResponseTime.Value), "ticks"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, Convert.ToInt64(_endGetResponseTime.Value), "ticks"));
}
else
{
metrics.Add(new LongMetric(MetricName.beginGetResponseTime, timeStart.Ticks, "ticks"));
metrics.Add(new LongMetric(MetricName.endGetResponseTime, timeStop.Ticks, "ticks"));
}
}
metrics.Add(new LongMetric(MetricName.maxTotalResults, results.TotalResults, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, results.Count, "#"));
if (results.Count == 0)
{
if (results.TotalResults > 0 && !ignoreEmptyResult)
{
log.WarnFormat("[{1}] no entries for {2} whilst {0} totalResult. This seems to be an error.", results.TotalResults, Task.CurrentId, fd.Label);
metrics.Add(new ExceptionMetric(new InvalidOperationException("No entries found")));
}
}
else
{
try
{
Stopwatch sw3 = new Stopwatch();
sw3.Start();
metrics.AddRange(AnalyzeResults(results, fd));
sw3.Stop();
metrics.Add(new LongMetric(MetricName.analysisTime, sw3.ElapsedMilliseconds, "msec"));
}
catch (Exception e)
{
metrics.Add(new ExceptionMetric(e));
log.ErrorFormat("[{0}] < Analysis failed for results {2} : {1}", Task.CurrentId, e.Message, fd.Label);
log.Debug(e.StackTrace);
}
}
}
return new TestUnitResult(metrics, fd);
}).Result);
}
private void PrintOpenSearchDescription(string arg)
{
Match argMatch = argRegex.Match(arg);
string type = argMatch.Groups[1].Value;
string paramName = (argMatch.Groups[2].Success ? argMatch.Groups[3].Value : null);
bool typeFound = false;
bool paramFound = false;
dataModelParameters = PrepareDataModelParameters();
dataModel = DataModel.CreateFromArgs(queryModelArg, dataModelParameters);
OpenSearchableFactorySettings settings = new OpenSearchableFactorySettings(ose);
settings.MaxRetries = retryAttempts;
List <Uri> baseUrls = InitializeUrl();
IOpenSearchable entity = dataModel.CreateOpenSearchable(baseUrls, queryFormatArg, ose, netCreds, settings);
OpenSearchDescription osd = entity.GetOpenSearchDescription();
using (Stream outputStream = InitializeOutputStream()) {
using (StreamWriter sw = new StreamWriter(outputStream)) {
if (type == "types")
{
sw.WriteLine("Formats");
sw.WriteLine("{0,-10}{1,-10}{2,-60}", "Format", "Supported", "MIME type");
sw.WriteLine("{0,-10}{1,-10}{2,-60}", "======", "=========", "=========");
foreach (OpenSearchDescriptionUrl url in osd.Url)
{
if (url.Relation == "self")
{
continue;
}
bool isFormat = false;
foreach (IOpenSearchEngineExtension osee in ose.Extensions.Values)
{
if (osee.DiscoveryContentType == url.Type)
{
isFormat = true;
break;
}
}
string urlShortType = GetUrlShortType(url.Type);
sw.WriteLine("{0,-14}{1,-6}{2,-40}", urlShortType == url.Type ? "-" : urlShortType, isFormat ? "*" : " ", url.Type);
}
}
else
{
foreach (OpenSearchDescriptionUrl url in osd.Url)
{
if (url.Relation == "self")
{
continue;
}
string urlShortType = GetUrlShortType(url.Type);
if (type != "full" && type != urlShortType)
{
continue;
}
typeFound = true;
bool isFormat = false;
foreach (IOpenSearchEngineExtension osee in ose.Extensions.Values)
{
if (osee.DiscoveryContentType == url.Type)
{
isFormat = true;
break;
}
}
sw.Write("Search URL type: {0}", url.Type);
if (isFormat)
{
sw.Write(" (format: {0})", urlShortType);
}
sw.WriteLine();
int qmPos = url.Template.IndexOf('?');
if (qmPos == -1)
{
continue;
}
string queryString = url.Template.Substring(qmPos + 1);
if (paramName == null)
{
sw.WriteLine("Parameters");
sw.WriteLine("{0,-22}{1,-40} M O R P S (mandatory, options, range, pattern, step)", "Name", "Description/title");
sw.WriteLine("{0,-22}{1,-40} ---------", "----", "-----------------");
}
MatchCollection paramMatches = paramRegex.Matches(queryString);
foreach (Match paramMatch in paramMatches)
{
string name = paramMatch.Groups[1].Value;
string identifier = paramMatch.Groups[2].Value;
bool mandatory = !paramMatch.Groups[3].Success;
string title = GetParameterDescription(identifier);
bool options = false;
bool range = false;
bool pattern = false;
bool step = false;
OpenSearchDescriptionUrlParameter param = null;
if (url.Parameters != null)
{
foreach (OpenSearchDescriptionUrlParameter p in url.Parameters)
{
if (p.Name == name)
{
param = p;
}
}
}
if (param != null)
{
title = param.Title;
options = param.Options != null && param.Options.Count != 0;
range = param.Maximum != null || param.MinInclusive != null || param.MaxInclusive != null || param.MinExclusive != null || param.MaxExclusive != null;
pattern = param.Pattern != null;
step = param.Step != null;
}
if (paramName == null)
{
if (title != null && title.Length > 40)
{
title = String.Format("{0}...", title.Substring(0, 37));
}
sw.WriteLine("- {0,-20}{1,-40} {2,-2}{3,-2}{4,-2}{5,-2}{6,-2}",
name,
title,
mandatory ? "M" : "-",
options ? "O" : "-",
range ? "R" : "-",
pattern ? "P" : "-",
step ? "S" : "-"
);
}
if (type != "full" && paramName != name)
{
continue;
}
paramFound = true;
if (paramName != null)
{
sw.WriteLine("- Parameter: {0}", name);
sw.WriteLine(" Description/title: {0}", title);
}
if (options)
{
sw.WriteLine(" Options:");
sw.WriteLine(" {0,-22} {1,-40}", "Value", "Label/description");
sw.WriteLine(" {0,-22} {1,-40}", "-----", "-----------------");
foreach (OpenSearchDescriptionUrlParameterOption o in param.Options)
{
sw.WriteLine(" - {0,-20} {1,-40}", o.Value, o.Label);
}
}
if (range)
{
string min = (param.MinExclusive != null ? param.MinExclusive : param.MinInclusive != null ? param.MinInclusive : param.Minimum);
string max = (param.MaxExclusive != null ? param.MaxExclusive : param.MaxInclusive != null ? param.MaxInclusive : param.Maximum);
sw.WriteLine(" Range: {0} {2} value {3} {1}", min, max, param.MinExclusive == null ? "<=" : "<", param.MaxExclusive == null ? "<=" : "<");
}
if (pattern)
{
sw.WriteLine(" Pattern: {0}", param.Pattern);
}
if (step)
{
sw.WriteLine(" Step: {0}", param.Step);
}
}
sw.WriteLine();
//sw.WriteLine("URL {0} {1} {2}", url.Type, url.Relation, url.Template);
}
}
sw.Close();
}
}
if (!typeFound && type != "types" && type != "full")
{
log.Error("URL Type not found");
}
else if (!paramFound && paramName != null)
{
log.Error("Parameter not found");
}
}
/// <summary>
/// Query the specified IOpenSearchable entity with specific parameters and returns result in native format.
/// </summary>
/// <param name="entity">Entity.</param>
/// <param name="parameters">Parameters.</param>
public IOpenSearchResultCollection Query(IOpenSearchable entity, NameValueCollection parameters)
{
// Transform action to invoke
QuerySettings querySettings;
// 1) Find the plugin to match with urlTemplates
querySettings = entity.GetQuerySettings(this);
if (querySettings == null)
throw new ImpossibleSearchException(String.Format("No engine extension to query {0}", entity.Identifier));
// 2) Create the request
OpenSearchRequest request = entity.Create(querySettings, parameters);
// 3) Apply the pre-search functions
ApplyPreSearchFilters(ref request);
// 4) Perform the Search
IOpenSearchResponse response = request.GetResponse();
response.Entity = entity;
// 5) Apply the pre-search functions
ApplyPostSearchFilters(request, ref response);
// 6) Read the response
var results = querySettings.ReadNative.Invoke(response);
results.Parameters = request.Parameters;
results.LastUpdatedTime = DateTime.Now;
// 7) Apply post search filters
entity.ApplyResultFilters(request, ref results, results.ContentType);
// 8 Create the container
ApplyOpenSearchElements(ref results, request, response);
return results;
}
/// <summary>
/// Query the specified IOpenSearchable entity with specific parameters and result with the specific result type.
/// </summary>
/// <param name="entity">Entity.</param>
/// <param name="parameters">Parameters.</param>
/// <param name="resultType">Result type.</param>
public IOpenSearchResultCollection Query(IOpenSearchable entity, NameValueCollection parameters, Type resultType)
{
// Transform action to invoke
QuerySettings querySettings;
// 1) Find the best match between urlTemplates and result format
querySettings = entity.GetQuerySettings(this);
if (querySettings == null)
throw new ImpossibleSearchException(String.Format("No engine extension to query {0} in order to return {1}", entity.Identifier, resultType.FullName));
// 2) Create the request
OpenSearchRequest request = entity.Create(querySettings, parameters);
// 3) Apply the pre-search functions
ApplyPreSearchFilters(ref request);
// 4) Perform the Search
IOpenSearchResponse response = request.GetResponse();
response.Entity = entity;
// 5) Apply the pre-search functions
ApplyPostSearchFilters(request, ref response);
// 6) Transform the response
IOpenSearchResultCollection results = querySettings.ReadNative.Invoke(response);
results.Parameters = request.Parameters;
results.LastUpdatedTime = DateTime.Now;
IOpenSearchEngineExtension osee = GetFirstExtensionByTypeAbility(resultType);
// 7) Apply post search filters
entity.ApplyResultFilters(request, ref results, osee.DiscoveryContentType);
// 8) Change format
IOpenSearchResultCollection newResults = osee.CreateOpenSearchResultFromOpenSearchResult(results);
// 9) Create Result container
ApplyOpenSearchElements(ref newResults, request, response);
return newResults;
}
/// <summary> /// Applies the result filters. /// </summary> /// <param name="osr">Osr.</param> /// <param name="originalParameters">Original parameters.</param> public abstract void ApplyResultFilters(ref IOpenSearchResultCollection osr, NameValueCollection originalParameters, IOpenSearchable entity);
public override IOpenSearchable CreateOpenSearchable(IEnumerable <Uri> baseUrls, string queryFormatArg, OpenSearchEngine ose, IEnumerable <NetworkCredential> netCreds, OpenSearchableFactorySettings settings)
{
List <IOpenSearchable> entities = new List <IOpenSearchable>();
IOpenSearchEngineExtension ext;
if (string.IsNullOrEmpty(queryFormatArg))
{
ext = ose.GetExtensionByExtensionName("atom");
}
else
{
ext = ose.GetExtensionByExtensionName(queryFormatArg);
}
for (int i = 0; i < baseUrls.Count(); i++)
{
var url = baseUrls.ElementAt(i);
settings.Credentials = netCreds == null ? null : netCreds.ElementAt(i);
IOpenSearchable e = null;
// QC Sentinel1 case
if (url.Host == "qc.sentinel1.eo.esa.int")
{
log.DebugFormat("QC Sentinel1 source. Trying to get the earthobservation profile");
e = new Sentinel1QcOpenSearchable(url, settings.OpenSearchEngine);
entities.Add(e);
continue;
}
// USGS case
if (url.Host == "earthexplorer.usgs.gov")
{
log.DebugFormat("Usgs source. Trying to get the earthobservation profile");
// usgsOpenSearchable
e = new Terradue.OpenSearch.Usgs.UsgsOpenSearchable(url, settings);
entities.Add(e);
continue;
}
e = OpenSearchFactory.FindOpenSearchable(settings, url, ext.DiscoveryContentType);
if (!e.DefaultMimeType.Contains("profile=http://earth.esa.int/eop"))
{
try {
e = OpenSearchFactory.FindOpenSearchable(settings, url, "application/atom+xml; profile=http://earth.esa.int/eop/2.1");
} catch (InvalidOperationException) {
e = OpenSearchFactory.FindOpenSearchable(settings, url, "application/atom+xml");
}
if (!e.DefaultMimeType.Contains("xml"))
{
throw new InvalidOperationException("No Url in the OpenSearch Description Document that could fit the EOP data model");
}
}
// Fedeo case
if (url.Host == "fedeo.esa.int" && e.DefaultMimeType == "application/atom+xml" && e is Terradue.OpenSearch.GenericOpenSearchable)
{
log.DebugFormat("Fedeo source. Trying to get the earthobservation profile");
e = FedeoOpenSearchable.CreateFrom(url, settings.OpenSearchEngine);
}
// Cwic case
if (url.Host == "cwic.wgiss.ceos.org" && e.DefaultMimeType == "application/atom+xml" && e is Terradue.OpenSearch.GenericOpenSearchable)
{
log.DebugFormat("Cwic source. Trying to get the earthobservation profile");
e = CwicOpenSearchable.CreateFrom((Terradue.OpenSearch.GenericOpenSearchable)e, settings.OpenSearchEngine);
}
entities.Add(e);
}
IOpenSearchable entity;
OpenSearchableFactorySettings settings2 = new OpenSearchableFactorySettings(ose);
if (entities.Count > 1)
{
entity = new MultiGenericOpenSearchable(entities, settings2);
}
else
{
entity = entities[0];
}
return(entity);
}
/// <summary>
/// Create an OpenSearchRequest based on the IOpenSearchable entity, type and parameters.
/// </summary>
/// <description>
/// This method is the basic default implementation for creating OpenSearchRequest. It supports only 2 schemes:
/// 1) file:// that will create a FileOpenSearchRequest
/// 2) http:// that will create an HttpOpenSearchRequest
/// <param name="entity">IOpenSearchable Entity.</param>
/// <param name="type">MimeType of the request.</param>
/// <param name="parameters">Parameters of the request.</param>
public static OpenSearchRequest Create(IOpenSearchable entity, QuerySettings querySettings, NameValueCollection parameters)
{
OpenSearchDescription osd = entity.GetOpenSearchDescription();
OpenSearchDescriptionUrl url = OpenSearchFactory.GetOpenSearchUrlByType(osd, querySettings.PreferredContentType);
if (url == null) throw new InvalidOperationException(string.Format("Could not find a URL template for entity {0} with type {1}", entity.Identifier, querySettings.PreferredContentType));
OpenSearchUrl queryUrl = OpenSearchFactory.BuildRequestUrlForTemplate(url, parameters, entity.GetOpenSearchParameters(querySettings.PreferredContentType), querySettings);
OpenSearchRequest request = null;
switch (queryUrl.Scheme) {
case "http":
case "https":
request = new HttpOpenSearchRequest(queryUrl, querySettings.PreferredContentType);
((HttpOpenSearchRequest)request).TimeOut = 60000;
break;
case "file":
request = new FileOpenSearchRequest(queryUrl, querySettings.PreferredContentType);
break;
}
request.OriginalParameters = parameters;
return request;
}
NameValueCollection PrepareQueryParameters(IOpenSearchable entity)
{
NameValueCollection nvc = new NameValueCollection();
totalResults = 0;
foreach (var parameter in parameterArgs)
{
Match matchParamDef = Regex.Match(parameter, @"^(.*)=(.*)$");
// if martch is successful
if (matchParamDef.Success)
{
// TODO filter and convert query param
if (matchParamDef.Groups[1].Value == "count")
{
if (matchParamDef.Groups[2].Value == "unlimited")
{
nvc.Add(matchParamDef.Groups[1].Value, pagination.ToString());
totalResults = int.MaxValue;
continue;
}
totalResults = int.Parse(matchParamDef.Groups[2].Value);
if (totalResults > pagination)
{
nvc.Add(matchParamDef.Groups[1].Value, pagination.ToString());
continue;
}
}
nvc.Add(matchParamDef.Groups[1].Value, matchParamDef.Groups[2].Value);
}
}
if (totalResults == 0)
{
totalResults = pagination;
nvc.Add("count", pagination.ToString());
}
var remoteParams = entity.GetOpenSearchParameters(entity.DefaultMimeType);
if (remoteParams["do"] != null && nvc["do"] == null)
{
nvc["do"] = Dns.GetHostName();
}
dataModel.SetQueryParameters(nvc);
return nvc;
}
/// <summary>
/// Replaces the enclosure links.
/// </summary>
/// <param name="entity">Entity.</param>
/// <param name="parameters">Parameters.</param>
/// <param name="osr">Osr.</param>
/// <param name="entryEnclosureLinkTemplate">Entry enclosure link template.</param>
/// <param name="contentType">Content type.</param>
public void ReplaceEnclosureLinks(IOpenSearchable entity, NameValueCollection parameters, IOpenSearchResultCollection osr, string contentType)
{
if (!string.IsNullOrEmpty(parameters["do"]))
{
bool strict = false, isDomain = false;
string origin = parameters["do"];
if (parameters["do"].StartsWith("[") && parameters["do"].EndsWith("]"))
{
strict = true;
origin = parameters["do"].TrimStart('[').TrimEnd(']');
}
try
{
Dns.GetHostAddresses(origin);
isDomain = true;
}
catch { }
foreach (IOpenSearchResultItem item in osr.Items)
{
foreach (var enclosureLink in item.Links.Where(l => l.RelationshipType == "enclosure").ToArray())
{
if (origin.Contains("terradue") || isDomain)
{
SyndicationLink newEnclosureLink = DataGatewayFactory.SubstituteEnclosure(enclosureLink, openSearchable, item);
if (newEnclosureLink != null)
{
item.Links.Insert(item.Links.IndexOf(enclosureLink), newEnclosureLink);
item.Links.Remove(enclosureLink);
}
else if (strict)
{
item.Links.Remove(enclosureLink);
}
}
}
item.ElementExtensions = new SyndicationElementExtensionCollection(
item.ElementExtensions.Select <SyndicationElementExtension, SyndicationElementExtension>(ext =>
{
if (ext.OuterName != "offering" || ext.OuterNamespace != "http://www.opengis.net/owc/1.0")
{
return(ext);
}
var offering = (OwcOffering)OwcContextHelper.OwcOfferingSerializer.Deserialize(ext.GetReader());
if (offering.Contents != null)
{
foreach (var content in offering.Contents)
{
if (content.Url != null)
{
var newUrl = DataGatewayFactory.SubstituteUrlApi(content.Url, openSearchable, item);
if (newUrl != null)
{
content.Url = newUrl;
}
}
}
}
return(new SyndicationElementExtension(offering.CreateReader()));
}));
}
}
}
NameValueCollection ResolveParameters(NameValueCollection nameValueCollection, IOpenSearchable entity)
{
string contentType = entity.DefaultMimeType;
NameValueCollection osdParam = entity.GetOpenSearchParameters(contentType);
NameValueCollection osdRevParams = OpenSearchFactory.ReverseTemplateOpenSearchParameters(osdParam);
NameValueCollection parameters = new NameValueCollection();
foreach (string key in nameValueCollection.AllKeys)
{
if (osdRevParams[key] != null)
{
foreach (var id in osdRevParams.GetValues(key))
parameters.Set(id, nameValueCollection[key]);
}
else {
parameters.Set(key, nameValueCollection[key]);
}
}
return parameters;
}
public override IOpenSearchable[] GetOpenSearchableArray()
{
List <IOpenSearchable> osResources = new List <IOpenSearchable>(Resources.Count);
var settings = MasterCatalogue.OpenSearchFactorySettings;
string apikey = null;
string t2userid = null;
if (context.UserId != 0)
{
var user = UserTep.FromId(context, context.UserId);
apikey = user.GetSessionApiKey();
t2userid = user.TerradueCloudUsername;
}
foreach (RemoteResource res in Resources)
{
OpenSearchableFactorySettings specsettings = (OpenSearchableFactorySettings)settings.Clone();
// For Terradue resources, use the API key
if (res.Location.StartsWith(context.GetConfigValue("catalog-baseurl")) && !string.IsNullOrEmpty(apikey))
{
specsettings.Credentials = new System.Net.NetworkCredential(t2userid, apikey);
}
try {
IOpenSearchable entity = null;
try {
entity = OpenSearchFactory.FindOpenSearchable(specsettings, new Uri(res.Location), "application/atom+xml");
}catch (Exception e) {
if (!res.Location.StartsWith(context.GetConfigValue("catalog-baseurl")))
{
try {
entity = OpenSearchFactory.FindOpenSearchable(specsettings, new Uri(res.Location));
} catch (Exception ex) {
var feed = new AtomFeed();
feed.Id = "Exception";
feed.ElementExtensions.Add(
new SyndicationElementExtension("exception", "", new ExceptionMessage {
Message = ex.Message, Source = res.Location, HelpLink = ex.HelpLink
})
);
entity = new ExceptionOpensearchable(feed);
}
}
else
{
var feed = new AtomFeed();
feed.Id = "Exception";
feed.ElementExtensions.Add(
new SyndicationElementExtension("exception", "", new ExceptionMessage {
Message = e.Message, Source = res.Location, HelpLink = e.HelpLink
})
);
entity = new ExceptionOpensearchable(feed);
}
}
if (entity != null)
{
osResources.Add(entity);
}
}
catch (Exception e)
{
context.LogError(this, e.Message);
}
}
return(osResources.ToArray());
}
