/// <summary>
/// Get number of species observations that matches
/// provided species observation search criteria.
/// </summary>
/// <param name="context">Web service request context.</param>
/// <param name="searchCriteria">Species observation search criteria.</param>
/// <param name="coordinateSystem">Coordinate system used in geometry search criteria.</param>
/// <returns>
/// Number of species observations that matches
/// provided species observation search criteria.
/// </returns>
public Int64 GetSpeciesObservationCountBySearchCriteriaElasticsearch(WebServiceContext context,
WebSpeciesObservationSearchCriteria searchCriteria,
WebCoordinateSystem coordinateSystem)
{
DocumentCountResponse speciesObservationCountResponse;
StringBuilder filter;
// Check users access rights.
WebServiceData.AuthorizationManager.CheckAuthorization(context, AuthorityIdentifier.Sighting);
// Check that data is valid.
CheckData(context, searchCriteria, coordinateSystem);
// Get species observation filter.
filter = new StringBuilder();
filter.Append("{");
filter.Append(searchCriteria.GetFilter(context, false));
filter.Append("}");
// Get species observation count.
using (ElasticsearchSpeciesObservationProxy elastisearch = WebServiceData.DatabaseManager.GetElastisearchSpeciesObservationProxy())
{
speciesObservationCountResponse = elastisearch.GetSpeciesObservationCount(filter.ToString());
}
return(speciesObservationCountResponse.DocumentCount);
}
public void CreateIndex()
{
ElasticsearchSpeciesObservationProxy elasticsearch;
elasticsearch = new ElasticsearchSpeciesObservationProxy("swedish_species_observation" + "_2016_10_14");
elasticsearch.CreateIndex();
}
private ElasticsearchSpeciesObservationProxy GetElasticsearch()
{
if (mElasticsearch.IsNull())
{
mElasticsearch = new ElasticsearchSpeciesObservationProxy("swedish_species_observation");
}
return(mElasticsearch);
}
public void DeleteIndex()
{
ElasticsearchSpeciesObservationProxy elasticsearch;
Configuration.InstallationType = InstallationType.LocalTest;
elasticsearch = new ElasticsearchSpeciesObservationProxy("swedish_species_observation" + "_2016_01_26");
elasticsearch.DeleteIndex();
}
/// <summary>
/// Check that species observation search criteria is valid.
/// And also changes the searchCriteria by converting coordinates and adding taxonid's
/// This method should only be used together with Elasticsearch.
/// </summary>
/// <param name="context">Web service request context.</param>
/// <param name="searchCriteria">Search criteria.</param>
/// <param name="coordinateSystem">
/// Coordinate system used in geometry search criteria
/// and returned species observations.
/// </param>
public void CheckData(WebServiceContext context,
WebSpeciesObservationSearchCriteria searchCriteria,
WebCoordinateSystem coordinateSystem)
{
Dictionary <String, WebSpeciesObservationField> mapping;
coordinateSystem.CheckData();
searchCriteria.CheckNotNull("searchCriteria");
using (ElasticsearchSpeciesObservationProxy elastisearch = WebServiceData.DatabaseManager.GetElastisearchSpeciesObservationProxy())
{
mapping = GetMapping(context, elastisearch);
}
searchCriteria.CheckData(context, true, mapping);
searchCriteria.Polygons = ConvertToElasticSearchCoordinates(context,
searchCriteria.Polygons,
searchCriteria.RegionGuids,
coordinateSystem);
searchCriteria.TaxonIds = WebSpeciesObservationServiceData.TaxonManager.GetTaxonIds(context, searchCriteria, true);
}
/// <summary>
/// Check that all species observation fields has been
/// mapped in Elasticsearch.
/// </summary>
/// <param name="context"> Web service request context.</param>
/// <param name="speciesObservation">Species observation.</param>
/// <param name="elasticsearch">Proxy to Elasticsearch.</param>
public void CheckMappingElasticsearch(WebServiceContext context,
WebSpeciesObservation speciesObservation,
ElasticsearchSpeciesObservationProxy elasticsearch)
{
Boolean isMappingUpdated;
Dictionary <String, WebSpeciesObservationField> mapping;
String cacheKey, fieldName, indexType, newMapping;
isMappingUpdated = false;
if (speciesObservation.IsNotNull() &&
speciesObservation.Fields.IsNotEmpty())
{
mapping = GetMapping(context, elasticsearch);
foreach (WebSpeciesObservationField field in speciesObservation.Fields)
{
fieldName = field.GetFieldName();
if (!mapping.ContainsKey(fieldName))
{
// Add mapping for new field.
switch (field.Type)
{
case WebDataType.Boolean:
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"boolean\"}" +
"}}";
break;
case WebDataType.DateTime:
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"date\", \"format\": \"dateOptionalTime\"}," +
"\"" + fieldName + "_DatePartOnly\": {\"type\": \"string\", \"index\": \"not_analyzed\"}," +
"\"" + fieldName + "_DayOfMonth\": {\"type\": \"byte\"}," +
"\"" + fieldName + "_DayOfYear\": {\"type\": \"short\"}," +
"\"" + fieldName + "_MonthOfYear\": {\"type\": \"byte\"}," +
"\"" + fieldName + "_WeekOfYear\": {\"type\": \"byte\"}," +
"\"" + fieldName + "_Year\": {\"type\": \"short\"}," +
"\"" + fieldName + "_YearAndMonth\": {\"type\": \"string\", \"index\": \"not_analyzed\"}," +
"\"" + fieldName + "_YearAndWeek\": {\"type\": \"string\", \"index\": \"not_analyzed\"}" +
"}}";
break;
case WebDataType.Float64:
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"double\"}" +
"}}";
break;
case WebDataType.Int32:
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"integer\"}" +
"}}";
break;
case WebDataType.Int64:
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"long\"}" +
"}}";
break;
case WebDataType.String:
switch (fieldName)
{
case "DarwinCore_DatasetName":
case "DarwinCore_Owner":
case "Occurrence_RecordedBy":
case "DarwinCore_ReportedBy":
// Aggregations are done on these fields in AnalysisService.
indexType = "not_analyzed";
break;
default:
indexType = "no";
break;
}
newMapping = "{\"properties\": {" +
"\"" + fieldName + "\": {\"type\": \"string\", \"index\": \"" + indexType + "\"}," +
"\"" + fieldName + "_Lowercase" + "\": {\"type\": \"string\", \"index\": \"not_analyzed\"}" +
"}}";
break;
default:
throw new Exception("Not handled field data type = " + field.Type);
}
// ReSharper disable once PossibleNullReferenceException
elasticsearch.UpdateSpeciesObservationMapping(newMapping);
isMappingUpdated = true;
}
}
}
if (isMappingUpdated)
{
// Wait a while in order to make sure that
// Elasticsearch has saved updated mapping.
Thread.Sleep(6000);
// Update cached mapping information.
cacheKey = Settings.Default.SpeciesObservationFieldMappingCacheKey;
context.RemoveCachedObject(cacheKey);
GetMapping(context, elasticsearch);
}
}
/// <summary>
/// Get mapping for species observation fields.
/// </summary>
/// <param name="context">Web service request context. </param>
/// <param name="elasticsearch">Proxy to Elasticsearch.</param>
/// <returns>Mapping for species observation fields.</returns>
public Dictionary <String, WebSpeciesObservationField> GetMapping(WebServiceContext context,
ElasticsearchSpeciesObservationProxy elasticsearch)
{
Dictionary <String, WebSpeciesObservationField> mapping;
FieldDefinitionList fieldDefinitions;
String cacheKey;
String[] splitField;
WebSpeciesObservationField field;
// Get data from cache.
cacheKey = Settings.Default.SpeciesObservationFieldMappingCacheKey;
mapping = (Dictionary <String, WebSpeciesObservationField>)context.GetCachedObject(cacheKey);
if (mapping.IsNull())
{
// Get data from Elasticsearch.
mapping = new Dictionary <String, WebSpeciesObservationField>();
fieldDefinitions = elasticsearch.GetSpeciesObservationMapping();
if (fieldDefinitions.IsNotEmpty())
{
foreach (FieldDefinition fieldDefinition in fieldDefinitions)
{
if (1 <= fieldDefinition.Name.IndexOf('_'))
{
splitField = fieldDefinition.Name.Split('_');
field = new WebSpeciesObservationField();
field.ClassIdentifier = splitField[0];
field.PropertyIdentifier = splitField[1];
switch (fieldDefinition.DataType)
{
case "boolean":
field.Type = WebDataType.Boolean;
break;
case "byte":
field.Type = WebDataType.Int32;
break;
case "date":
field.Type = WebDataType.DateTime;
break;
case "double":
field.Type = WebDataType.Float64;
break;
case "integer":
field.Type = WebDataType.Int32;
break;
case "long":
field.Type = WebDataType.Int64;
break;
case "short":
field.Type = WebDataType.Int32;
break;
case "string":
field.Type = WebDataType.String;
break;
default:
throw new Exception("Not handled data type = " + fieldDefinition.DataType);
}
mapping[fieldDefinition.Name] = field;
////Debug.WriteLine(fieldDefinition.Name + " " +
//// field.ClassIdentifier + " " +
//// field.PropertyIdentifier + " " +
//// field.Type);
}
}
}
// Store data in cache.
context.AddCachedObject(cacheKey,
mapping,
DateTime.Now + new TimeSpan(1, 0, 0, 0),
CacheItemPriority.High);
}
return(mapping);
}
public void CheckDifference()
{
Dictionary <String, WebSpeciesObservationField> mapping;
DocumentFilterResponse documentFilterResponse;
ElasticsearchSpeciesObservationProxy elasticsearch;
Int32 index;
List <Int64> speciesObservationAdd, speciesObservationDelete,
speciesObservationIds, speciesObservationDifferenceIds;
List <WebSpeciesObservation> speciesObservations;
SpeciesObservationElasticsearch speciesObservationElasticsearch;
StringBuilder filter;
speciesObservationAdd = new List <Int64>();
speciesObservationDelete = new List <Int64>();
HarvestManager.GetSpeciesObservationDifference(GetContext(), speciesObservationAdd, speciesObservationDelete);
if (speciesObservationAdd.IsNotEmpty())
{
speciesObservationElasticsearch = WebSpeciesObservationServiceData.SpeciesObservationManager.GetSpeciesObservationElasticsearch(GetContext());
elasticsearch = new ElasticsearchSpeciesObservationProxy(speciesObservationElasticsearch.CurrentIndexName);
mapping = WebSpeciesObservationServiceData.SpeciesObservationManager.GetMapping(GetContext(), elasticsearch);
filter = new StringBuilder();
filter.Append("{");
filter.Append(" \"size\": " + speciesObservationAdd.Count);
filter.Append(", \"_source\" : {\"include\": [\"Occurrence_OccurrenceID\", \"DarwinCore_Id\"]}");
filter.Append(", \"filter\": { \"terms\": { \"DarwinCore_Id\":[");
filter.Append(speciesObservationAdd[0].WebToString());
for (index = 1; index < speciesObservationAdd.Count; index++)
{
filter.Append(", " + speciesObservationAdd[index].WebToString());
}
filter.Append("]}}}");
documentFilterResponse = elasticsearch.GetSpeciesObservations(filter.ToString());
if (documentFilterResponse.TimedOut)
{
throw new Exception("Method UpdateSpeciesObservationsElasticsearch() timed out!");
}
speciesObservations = WebSpeciesObservationServiceData.SpeciesObservationManager.GetSpeciesObservations(documentFilterResponse.DocumentsJson,
mapping);
if (speciesObservations.IsNotEmpty())
{
speciesObservationIds = new List <Int64>();
foreach (WebSpeciesObservation speciesObservationDeleteTemp in speciesObservations)
{
speciesObservationIds.Add(speciesObservationDeleteTemp.Fields.GetField(SpeciesObservationClassId.DarwinCore, SpeciesObservationPropertyId.Id).Value.WebParseInt64());
}
speciesObservationDifferenceIds = new List <Int64>();
foreach (Int64 speciesObservationId in speciesObservationAdd)
{
if (!(speciesObservationIds.Contains(speciesObservationId)))
{
speciesObservationDifferenceIds.Add(speciesObservationId);
Debug.WriteLine("Missing species observation id = " + speciesObservationId);
}
}
}
}
}
public void DeleteDifference()
{
Dictionary <String, WebSpeciesObservationField> mapping;
DocumentFilterResponse documentFilterResponse;
ElasticsearchSpeciesObservationProxy elasticsearch;
Int32 index;
List <Int64> speciesObservationAdd, speciesObservationDelete, speciesObservationIds;
List <WebSpeciesObservation> speciesObservations;
SpeciesObservationElasticsearch speciesObservationElasticsearch;
StringBuilder filter;
speciesObservationAdd = new List <Int64>();
speciesObservationDelete = new List <Int64>();
HarvestManager.GetSpeciesObservationDifference(GetContext(), speciesObservationAdd, speciesObservationDelete);
speciesObservationElasticsearch = WebSpeciesObservationServiceData.SpeciesObservationManager.GetSpeciesObservationElasticsearch(GetContext());
elasticsearch = new ElasticsearchSpeciesObservationProxy(speciesObservationElasticsearch.CurrentIndexName);
mapping = WebSpeciesObservationServiceData.SpeciesObservationManager.GetMapping(GetContext(), elasticsearch);
while (speciesObservationDelete.IsNotEmpty())
{
speciesObservationIds = new List <Int64>();
while (speciesObservationDelete.IsNotEmpty())
{
speciesObservationIds.Add(speciesObservationDelete[0]);
speciesObservationDelete.RemoveAt(0);
if (speciesObservationIds.Count >= 10000)
{
break;
}
}
filter = new StringBuilder();
filter.Append("{");
filter.Append(" \"size\": " + speciesObservationIds.Count);
filter.Append(", \"_source\" : {\"include\": [\"Occurrence_OccurrenceID\", \"DarwinCore_Id\", \"DarwinCore_DataProviderId\", \"Occurrence_CatalogNumber\"]}");
filter.Append(", \"filter\": { \"terms\": { \"DarwinCore_Id\":[");
filter.Append(speciesObservationIds[0].WebToString());
for (index = 1; index < speciesObservationIds.Count; index++)
{
filter.Append(", " + speciesObservationIds[index].WebToString());
}
filter.Append("]}}}");
documentFilterResponse = elasticsearch.GetSpeciesObservations(filter.ToString());
if (documentFilterResponse.TimedOut)
{
throw new Exception("Method UpdateSpeciesObservationsElasticsearch() timed out!");
}
speciesObservations = WebSpeciesObservationServiceData.SpeciesObservationManager.GetSpeciesObservations(documentFilterResponse.DocumentsJson,
mapping);
if (speciesObservations.IsNotEmpty())
{
DataTable deletedSpeciesObservationTable = new DataTable();
deletedSpeciesObservationTable.TableName = "DeletedSpeciesObservation";
deletedSpeciesObservationTable.Columns.Add("id", typeof(Int64));
deletedSpeciesObservationTable.Columns.Add("dataProviderId", typeof(Int32));
deletedSpeciesObservationTable.Columns.Add("catalogNumber", typeof(String));
deletedSpeciesObservationTable.Columns.Add("occuranceId", typeof(String));
foreach (WebSpeciesObservation speciesObservation in speciesObservations)
{
DataRow speciesObservationRow = deletedSpeciesObservationTable.NewRow();
speciesObservationRow[0] = speciesObservation.Fields.GetField(SpeciesObservationClassId.DarwinCore, SpeciesObservationPropertyId.Id).Value.WebParseInt64();
speciesObservationRow[1] = speciesObservation.Fields.GetField("DarwinCore", "DataProviderId").Value.WebParseInt32();
speciesObservationRow[2] = speciesObservation.Fields.GetField(SpeciesObservationClassId.Occurrence, SpeciesObservationPropertyId.CatalogNumber).Value;
speciesObservationRow[3] = speciesObservation.Fields.GetField(SpeciesObservationClassId.Occurrence, SpeciesObservationPropertyId.OccurrenceID).Value;
deletedSpeciesObservationTable.Rows.Add(speciesObservationRow);
}
GetContext().GetSpeciesObservationDatabase().AddTableData(GetContext(), deletedSpeciesObservationTable);
}
}
}
public ElasticsearchSpeciesObservationProxyTest()
{
mElasticsearch = null;
}
