public static Uri SubstituteUrlApi(Uri url, IOpenSearchable openSearchable, IOpenSearchResultItem item)
{
UriBuilder urib = new UriBuilder(url);
urib.Path = "";
urib.Query = null;
urib.Fragment = null;
var substUri = FindSubstituteUri(urib.Uri, true);
if (substUri == null)
{
return(null);
}
urib = new UriBuilder(substUri);
if (openSearchable is SandboxOpenSearchable)
{
urib.Path += RewritePath(url.AbsolutePath, openSearchable);
}
else
{
urib.Path += RewriteExternalPath(url);
}
return(urib.Uri);
}
public List <string> GetMetadataForItem(List <string> metadataPaths, IOpenSearchResultItem item)
{
List <string> metadataItems = new List <string>();
if (osr != null)
{
foreach (var metadata in metadataPaths)
{
string path = metadata;
string specifier = null;
if (metadata.Contains(":"))
{
path = metadata.Split(':')[0];
specifier = metadata.Split(':')[1];
}
if (!metadataExtractors.ContainsKey(path))
{
throw new NotSupportedException(string.Format("metadata extractor \"{0}\" not found in the data model", path));
}
metadataItems.Add(metadataExtractors[path].GetMetadata(item, specifier));
}
}
return(metadataItems);
}
private IOpenSearchResultItem FindReferenceItem(IOpenSearchResultItem item, IOpenSearchable os)
{
FiltersDefinition filters = new FiltersDefinition(item.Identifier);
filters.AddFilter("uid", "{http://a9.com/-/opensearch/extensions/geo/1.0/}uid", item.Identifier, item.Identifier, null, null);
var result = ose.Query(os, filters.GetNameValueCollection());
return(result.Items.FirstOrDefault());
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
var mode = Terradue.Metadata.EarthObservation.OpenSearch.Extensions.EarthObservationOpenSearchResultExtensions.FindOperationalMode(item);
if (mode.EndsWith("_SP") || mode.EndsWith("_DP"))
{
mode = mode.Replace("_SP", "").Replace("_DP", "");
}
return(mode);
}
public RdfXmlResult(IOpenSearchResultItem item) : base() {
if (item.Title != null)
Title = item.Title;
if (item.LastUpdatedTime.Ticks != 0)
LastUpdatedTime = item.LastUpdatedTime;
if (!string.IsNullOrEmpty(item.Identifier))
Identifier = item.Identifier;
elementExtensions = item.ElementExtensions;
links = item.Links;
this.SortKey = item.SortKey;
}
public static GeometryObject FindGeometry(IOpenSearchResultItem item)
{
GeometryObject savegeom = null;
if (item.ElementExtensions != null && item.ElementExtensions.Count > 0) {
foreach (var ext in item.ElementExtensions) {
XmlReader xr = ext.GetReader();
if ( string.IsNullOrEmpty(xr.LocalName))
xr.Read();
var name = xr.LocalName;
switch (xr.NamespaceURI) {
// 1) search for georss
case "http://www.georss.org/georss":
savegeom = Terradue.GeoJson.GeoRss.GeoRssHelper.Deserialize(xr).ToGeometry();
if (name != "box" && name != "point") {
return savegeom;
}
break;
// 2) search for georss10
case "http://www.georss.org/georss/10":
savegeom = GeoRss10Extensions.ToGeometry(GeoRss10Helper.Deserialize(xr));
if (name != "box" && name != "point") {
return savegeom;
}
break;
// 3) search for dct:spatial
case "http://purl.org/dc/terms/":
if (xr.LocalName == "spatial") {
xr.Read();
savegeom = WktExtensions.WktToGeometry(xr.Value);
if (!(savegeom is Point)) {
return savegeom;
}
}
break;
default:
continue;
}
}
}
return savegeom;
}
public static void AddOrReplaceEarthObservation(ServiceModel.Ogc.Eop20.EarthObservationType eo, IOpenSearchResultItem item)
{
if (eo != null)
{
foreach (var ext in item.ElementExtensions.ToArray())
{
if (ext.OuterName == "EarthObservation")
{
item.ElementExtensions.Remove(ext);
}
}
item.ElementExtensions.Add(Terradue.ServiceModel.Ogc.OgcHelpers.CreaterReader(eo));
}
}
public static string EntrySelfLinkTemplate(IOpenSearchResultItem item, OpenSearchDescription osd, string mimeType)
{
if (item == null)
return null;
string identifier = item.Identifier;
NameValueCollection nvc = new NameValueCollection();
nvc.Set("q", string.Format("_id:{0}", item.Identifier));
UriBuilder template = new UriBuilder(OpenSearchFactory.GetOpenSearchUrlByType(osd, mimeType).Template);
string[] queryString = Array.ConvertAll(nvc.AllKeys, key => string.Format("{0}={1}", key, nvc[key]));
template.Query = string.Join("&", queryString);
return template.ToString();
}
public static string EntrySelfLinkTemplate(IOpenSearchResultItem item, OpenSearchDescription osd, string mimeType)
{
if (item == null)
{
return(null);
}
string identifier = item.Identifier;
NameValueCollection nvc = new NameValueCollection();
nvc.Set("q", string.Format("_id:{0}", item.Identifier));
UriBuilder template = new UriBuilder(OpenSearchFactory.GetOpenSearchUrlByType(osd, mimeType).Template);
string[] queryString = Array.ConvertAll(nvc.AllKeys, key => string.Format("{0}={1}", key, nvc[key]));
template.Query = string.Join("&", queryString);
return(template.ToString());
}
public FeatureResult ImportItem(IOpenSearchResultItem result)
{
if (result == null)
throw new ArgumentNullException("result");
FeatureResult feature;
XmlDocument doc = new XmlDocument();
List<XmlElement> elements = new List<XmlElement>();
foreach (var ext in result.ElementExtensions) {
XmlElement element = ext.GetObject<XmlElement>();
elements.Add(element);
}
XmlElement geometry = ImportUtils.FindGmlGeometry(elements.ToArray());
if (geometry != null)
feature = new FeatureResult(Terradue.GeoJson.Geometry.GeometryFactory.GmlToFeature(geometry));
else {
geometry = util.FindDctSpatialGeometry(elements.ToArray());
if (geometry != null)
feature = new FeatureResult(Terradue.GeoJson.Geometry.GeometryFactory.WktToFeature(geometry.InnerXml));
else
feature = new FeatureResult(Terradue.GeoJson.Geometry.GeometryFactory.WktToFeature(null));
}
NameValueCollection namespaces = null;
XmlNamespaceManager xnsm = null;
string prefix = "";
feature.Id = result.Id;
feature.Identifier = result.identifier;
if (result.Date != null)
feature.Properties.Add(prefix + "published", result.Date.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'"));
if (result.Links != null && result.Links.Count > 0) {
feature.Links = result.Links;
}
if (result.Title != null)
feature.Title = result.Title;
return feature;
}
protected virtual void CreateDownloadRequestAndEnqueue(IOpenSearchResultItem item)
{
log.DebugFormat("Creating download request for {0}...", item.Identifier);
try
{
IAssetAccess enclosureAccess = target.Wrapper.GetEnclosureAccess(item);
if (enclosureAccess != null)
{
if (Convert.ToInt64(enclosureAccess.TotalSize) > target.TargetSiteConfig.MaxDownloadSize)
{
throw new Exception(String.Format("Product file too large ({0} > {1}) for {2}", enclosureAccess.TotalSize, target.TargetSiteConfig.MaxDownloadSize, item.Identifier));
}
downloadRequests.Enqueue(enclosureAccess);
log.DebugFormat("OK --> {0}", enclosureAccess.Uri);
}
}
catch (Exception e)
{
log.WarnFormat("NOT OK: {0}", e.Message);
log.Debug(e.StackTrace);
}
}
public new static GenericJsonItem FromOpenSearchResultItem(IOpenSearchResultItem result)
{
if (result is GenericJsonItem)
{
return((GenericJsonItem)result);
}
GenericJsonItem item = new GenericJsonItem();
item.Identifier = result.Identifier;
item.Date = result.Date;
foreach (SyndicationElementExtension ext in result.ElementExtensions)
{
XmlDocument doc = new XmlDocument();
doc.Load(ext.GetReader());
var obj = JsonConvert.DeserializeObject <Dictionary <string, object> >(JsonConvert.SerializeXmlNode(doc, Newtonsoft.Json.Formatting.None, false));
item.payload.Add(ext.OuterName, obj[ext.OuterName]);
}
return(item);
}
static SyndicationLink SubstituteSandboxEnclosure(SyndicationLink link, SandboxOpenSearchable sandboxOpenSearchable, IOpenSearchResultItem item)
{
UriBuilder urib = new UriBuilder(link.Uri.AbsoluteUri);
urib.Path = "";
urib.Query = null;
urib.Fragment = null;
var substUri = FindSubstituteUri(urib.Uri);
if (substUri == null)
{
return(null);
}
urib = new UriBuilder(substUri);
urib.Path += RewritePath(link.Uri.AbsolutePath, sandboxOpenSearchable);
return(new SyndicationLink(urib.Uri, "enclosure", link.Title + " via Data Gateway", link.MediaType, link.Length));
}
protected abstract DateTime[] GetStartAndStopTime(IOpenSearchResultItem item);
public string EntrySelfLinkTemplate(IOpenSearchResultItem item, OpenSearchDescription osd, string mimeType)
{
return(OpenSearchService.EntrySelfLinkTemplate(item, osd, mimeType));
}
public static void AddOrReplaceEarthObservation(ServiceModel.Ogc.Eop20.EarthObservationType eo, IOpenSearchResultItem item)
{
if (eo != null)
{
foreach (var ext in item.ElementExtensions.ToArray())
{
if (ext.OuterName == "EarthObservation")
{
item.ElementExtensions.Remove(ext);
}
}
item.ElementExtensions.Add(Terradue.ServiceModel.Ogc.OgcHelpers.CreateReader(eo));
}
}
public static new GenericJsonItem FromOpenSearchResultItem(IOpenSearchResultItem result)
{
if (result is GenericJsonItem)
return (GenericJsonItem)result;
GenericJsonItem item = new GenericJsonItem();
item.Identifier = result.Identifier;
item.Date = result.Date;
foreach (SyndicationElementExtension ext in result.ElementExtensions) {
XmlDocument doc = new XmlDocument();
doc.Load(ext.GetReader());
var obj = JsonConvert.DeserializeObject<Dictionary<string, object>>(JsonConvert.SerializeXmlNode(doc, Newtonsoft.Json.Formatting.None, false));
item.payload.Add(ext.OuterName, obj[ext.OuterName]);
}
return item;
}
public static List <WpsServiceOverview> GetWpsServiceOverviews(IfyContext context, IOpenSearchResultItem item)
{
var appUid = item.Identifier.Trim();
var appTitle = item.Title != null?item.Title.Text.Trim() : appUid;
var appIconLink = item.Links.FirstOrDefault(l => l.RelationshipType == "icon");
string appIcon = "";
if (appIconLink != null)
{
appIcon = appIconLink.Uri.AbsoluteUri;
}
var offerings = item.ElementExtensions.ReadElementExtensions <OwcOffering>("offering", OwcNamespaces.Owc, new System.Xml.Serialization.XmlSerializer(typeof(OwcOffering)));
var offering = offerings.First(p => p.Code == "http://www.opengis.net/spec/owc/1.0/req/atom/wps");
return(GetWpsServiceOverview(context, offering, appUid, appTitle, appIcon));
}
internal static FiltersDefinition GenerateFiltersDefinitionFromItem(string label, IOpenSearchResultItem item)
{
FiltersDefinition fd = new FiltersDefinition(string.Format("{0}-{1}", label, item.Identifier));
if (item == null)
{
return(fd);
}
fd.AddFilter("uid", "{http://a9.com/-/opensearch/extensions/geo/1.0/}uid", item.Identifier, item.Identifier, null, null);
var platform = item.FindPlatformShortName();
if (!string.IsNullOrEmpty(platform))
{
fd.AddFilter("missionName", "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform", platform, platform, null, null);
}
var instrument = item.FindInstrumentShortName();
if (!string.IsNullOrEmpty(instrument))
{
fd.AddFilter("instrument", "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument", instrument, instrument, null, null);
}
var productType = item.FindProductType();
if (!string.IsNullOrEmpty(productType))
{
fd.AddFilter("productType", "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType", productType, productType, null, null);
}
var processingLevel = item.FindProcessingLevel();
if (!string.IsNullOrEmpty(processingLevel))
{
fd.AddFilter("processingLevel", "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel", processingLevel, processingLevel, null, null);
}
try
{
var startDate = item.FindStartDate();
if (startDate != DateTime.MinValue)
{
fd.AddFilter("sensingStart", "{http://a9.com/-/opensearch/extensions/time/1.0/}start", startDate.ToString("O"), string.Format("starting at {0}", startDate.ToString()), null, null);
}
var endDate = item.FindEndDate();
if (endDate != DateTime.MinValue)
{
fd.AddFilter("sensingEnd", "{http://a9.com/-/opensearch/extensions/time/1.0/}end", endDate.ToString("O"), string.Format("ending at {0}", endDate.ToString()), null, null);
}
}
catch { }
return(fd);
}
public DateTime[] GetTimeCoverage(NameValueCollection searchParameters, IOpenSearchResultItem item)
{
string value = searchParameters["timeCover"];
if (string.IsNullOrEmpty(value)) {
return null;
}
try {
DateTime[] itemDates = GetStartAndStopTime(item);
var time = value.Split(',');
if ( time.Length != 2 ){
throw new FormatException(string.Format("Wrong format for param cor:time={0}. must be 2 timespan separated by ,", value));
}
TimeSpan[] timeSpans = Array.ConvertAll<string, TimeSpan>(time, t => TimeSpan.Parse(t));
return new DateTime[]{itemDates[0].Add(timeSpans[0]), itemDates[1].Add(timeSpans[1])};
} catch (Exception){
throw new FormatException(string.Format("Wrong format for param cor:time={0}. must be 2 timespan separated by ,", value));
}
}
public List<string> GetMetadataForItem(List<string> metadataPaths, IOpenSearchResultItem item)
{
List<string> metadataItems = new List<string>();
if (osr != null) {
foreach (var metadata in metadataPaths) {
string path = metadata;
string specifier = null;
if (metadata.Contains(":")) {
path = metadata.Split(':')[0];
specifier = metadata.Split(':')[1];
}
if (!metadataExtractors.ContainsKey(path))
throw new NotSupportedException(string.Format("metadata extractor \"{0}\" not found in the data model", path));
metadataItems.Add(metadataExtractors[path].GetMetadata(item, specifier));
}
}
return metadataItems;
}
string EntrySelfLinkTemplate(IOpenSearchResultItem item, OpenSearchDescription osd, string mimeType)
{
string identifier = item.Identifier;
return(EntrySelfLinkTemplate(identifier, osd, mimeType));
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return Terradue.Metadata.EarthObservation.OpenSearch.EarthObservationOpenSearchResultHelpers.FindPolarisationChannelsFromOpenSearchResultItem(item);
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return(Terradue.Metadata.EarthObservation.OpenSearch.Extensions.EarthObservationOpenSearchResultExtensions.FindParentIdentifier(item));
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return Terradue.Metadata.EarthObservation.OpenSearch.EarthObservationOpenSearchResultHelpers.FindPlatformShortNameFromOpenSearchResultItem(item);
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return(Terradue.Metadata.EarthObservation.OpenSearch.Extensions.EarthObservationOpenSearchResultExtensions.FindWrsLongitudeGrid(item));
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return(Terradue.Metadata.EarthObservation.OpenSearch.Extensions.EarthObservationOpenSearchResultExtensions.FindCloudCoverPercentage(item).ToString());
}
public string EntrySelfLinkTemplate(IOpenSearchResultItem item, OpenSearchDescription osd, string mimeType)
{
return OpenSearchService.EntrySelfLinkTemplate(item, osd, mimeType);
}
public static SyndicationLink SubstituteEnclosure(SyndicationLink link, IOpenSearchable os, IOpenSearchResultItem item)
{
if (os is SandboxOpenSearchable)
{
return(SubstituteSandboxEnclosure(link, os as SandboxOpenSearchable, item));
}
return(SubstituteExternalEnclosure(link));
}
public static FeatureResult FromOpenSearchResultItem(IOpenSearchResultItem result)
{
if (result == null)
throw new ArgumentNullException("result");
FeatureResult feature;
var geom = ImportUtils.FindGeometry(result);
if (geom != null)
feature = new FeatureResult(new Feature(geom, null));
else
feature = new FeatureResult();
feature.ElementExtensions = new SyndicationElementExtensionCollection(result.ElementExtensions);
feature.Id = result.Id;
feature.LastUpdatedTime = result.LastUpdatedTime;
feature.PublishDate = result.PublishDate;
feature.Summary = result.Summary;
feature.Content = result.Content;
feature.contributors = result.Contributors;
feature.authors = result.Authors;
feature.Title = result.Title;
feature.categories = result.Categories;
feature.Copyright = result.Copyright;
feature.Identifier = result.Identifier;
feature.Links = new Collection<SyndicationLink>(result.Links);
feature.sortKey = result.SortKey;
return feature;
}
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd)
{
List <IMetric> metrics = new List <IMetric>();
long errors = 0;
long validatedResults = 0;
log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId);
if (results.Count == 0 && results.TotalResults > 0)
{
log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis",
results.Count, results.TotalResults))));
return(metrics);
}
List <IMetric> _testCaseMetrics = new List <IMetric>();
int i = int.Parse(fd.Filters.FirstOrDefault(f => f.Key == "queryId").Value);
var os = queryFiltersTuple[i].Reference.Target.CreateOpenSearchableEntity();
List <double> avaLatencies = new List <double>();
foreach (IOpenSearchResultItem item in results.Items)
{
foreach (var filterDefinition in fd.GetFilters())
{
if (filterDefinition.ItemValidator != null)
{
if (filterDefinition.ItemValidator.Invoke(item))
{
continue;
}
log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId);
errors++;
}
if (filterDefinition.ResultsValidator != null)
{
if (filterDefinition.ResultsValidator.Invoke(results))
{
continue;
}
log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId);
errors++;
}
}
IOpenSearchResultItem ref_item = FindReferenceItem(item, os);
if (ref_item == null)
{
log.WarnFormat("item {0} not found in reference target", item.Identifier);
continue;
}
DateTimeOffset onlineDate = item.PublishDate;
var onlineDateStrings = item.ElementExtensions.ReadElementExtensions <string>("onlineDate", "http://www.terradue.com/");
if (onlineDateStrings != null && onlineDateStrings.Count() > 0)
{
DateTimeOffset.TryParse(onlineDateStrings.FirstOrDefault(), out onlineDate);
}
avaLatencies.Add(onlineDate.Subtract(ref_item.PublishDate).TotalSeconds);
validatedResults++;
}
metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#"));
metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#"));
if (avaLatencies.Count() > 0)
{
metrics.Add(new LongMetric(MetricName.avgDataAvailabilityLatency, (long)avaLatencies.Average(), "sec"));
metrics.Add(new LongMetric(MetricName.maxDataAvailabilityLatency, (long)avaLatencies.Max(), "sec"));
}
else
{
metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec"));
metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec"));
}
return(metrics);
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return Terradue.Metadata.EarthObservation.OpenSearch.EarthObservationOpenSearchResultHelpers.FindOrbitNumberFromOpenSearchResultItem(item);
}
private static void AddIMGOffering(Terradue.ServiceModel.Ogc.Eop21.EarthObservationType eo, IOpenSearchResultItem item)
{
Terradue.ServiceModel.Ogc.Eop21.BrowseInformationPropertyType[] bi = null;
if (eo.result != null && eo.result.Eop21EarthObservationResult.browse != null)
{
bi = eo.result.Eop21EarthObservationResult.browse;
}
if (bi != null)
{
foreach (var browse in bi)
{
if (browse.BrowseInformation.type != "img")
continue;
OwcOffering offering = new OwcOffering();
offering.Code = "http://www.opengis.net/spec/owc-atom/1.0/req/img";
offering.Contents = new OwcContent[1];
offering.Contents[0] = new OwcContent();
offering.Contents[0].Url = new Uri(browse.BrowseInformation.fileName.ServiceReference.href);
item.ElementExtensions.Add(offering, new XmlSerializer(typeof(OwcOffering)));
}
}
}
public string GetMetadata(IOpenSearchResultItem item, string specifier)
{
return Terradue.Metadata.EarthObservation.OpenSearch.EarthObservationOpenSearchResultHelpers.FindWrsLongitudeGridFromOpenSearchResultItem(item);
}
private static void AddIMGOffering(Terradue.ServiceModel.Ogc.Eop21.EarthObservationType eo, IOpenSearchResultItem item)
{
Terradue.ServiceModel.Ogc.Eop21.BrowseInformationPropertyType[] bi = null;
if (eo.result != null && eo.result.Eop21EarthObservationResult.browse != null)
{
bi = eo.result.Eop21EarthObservationResult.browse;
}
if (bi != null)
{
foreach (var browse in bi)
{
if (browse.BrowseInformation.type != "img")
{
continue;
}
OwcOffering offering = new OwcOffering();
offering.Code = "http://www.opengis.net/spec/owc-atom/1.0/req/img";
offering.Contents = new OwcContent[1];
offering.Contents[0] = new OwcContent();
offering.Contents[0].Url = new Uri(browse.BrowseInformation.fileName.ServiceReference.href);
item.ElementExtensions.Add(offering, new XmlSerializer(typeof(OwcOffering)));
}
}
}
