/// <summary> /// Update species observations. /// </summary> /// <param name="context">Web service request context.</param> /// <param name="speciesObservations">Updated species observations.</param> /// <param name="dataProvider">The dataProvider.</param> /// <param name="connectorServer">The connector server.</param> /// <param name="mappings">The mappings.</param> /// <param name="noOfUpdated">No of updated species observations.</param> /// <param name="noOfUpdatedErrors">No of updating errors.</param> public void UpdateSpeciesObservations(WebServiceContext context, List <WebData> speciesObservations, WebSpeciesObservationDataProvider dataProvider, IConnectorServer connectorServer, List <HarvestMapping> mappings, out int noOfUpdated, out int noOfUpdatedErrors) { noOfUpdated = 0; noOfUpdatedErrors = 0; ArtportalenProcess artportalenProcess = new ArtportalenProcess(); if (speciesObservations.IsNotEmpty()) { List <HarvestSpeciesObservation> updatedSpeciesObservations = new List <HarvestSpeciesObservation>(); for (Int32 index = speciesObservations.Count - 1; index >= 0; index--) { HarvestSpeciesObservation harvestSpeciesObservation = artportalenProcess.ProcessObservation(speciesObservations[index], mappings, context); updatedSpeciesObservations.Add(harvestSpeciesObservation); speciesObservations.RemoveAt(index); if (updatedSpeciesObservations.Count >= 10000) { break; } } connectorServer.UpdateSpeciesObservations(context, updatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); } }
/// <summary> /// Get information about species observations /// that has changed in the specified date range or /// from a specified changeId. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom"> /// Changed from date. /// </param> /// <param name="isChangedFromSpecified"> /// Is changed from specified. /// </param> /// <param name="changedTo"> /// Changed to date. /// </param> /// <param name="isChangedToSpecified"> /// Is changed to specified. /// </param> /// <param name="changeId"> /// From witch change id. /// </param> /// <param name="isChangeIdspecified"> /// Is changed id specified. /// </param> /// <param name="maxReturnedChanges"> /// Max number of observations returned. /// </param> /// <param name="mappings"> /// The mapping list. /// </param> /// <param name="context"> /// The web service context. /// </param> /// <param name="connectorServer"> /// The connector service. /// </param> public void GetSpeciesObservationChange(DateTime changedFrom, Boolean isChangedFromSpecified, DateTime changedTo, Boolean isChangedToSpecified, Int64 changeId, Boolean isChangeIdspecified, Int64 maxReturnedChanges, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); speciesObservationChange.UpdatedSpeciesObservations = new List <HarvestSpeciesObservation>(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; NorsProcess norsProcess = new NorsProcess(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebSpeciesObservationChange webSpeciesObservationChange = WebServiceProxy.NorsService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, changeId, isChangeIdspecified, maxReturnedChanges); // LOOP OVER CHANGEID // Denna verkar inte klara mer än ca 100 obsar så det behövs en loop över dessa... Int32 readSize = 0; Int64 currentChangeId = 0; int j = 0; while ((readSize++ < 50) && (currentChangeId < webSpeciesObservationChange.MaxChangeId)) { currentChangeId = webSpeciesObservationChange.MaxChangeId; context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, webSpeciesObservationChange.MaxChangeId); // CREATED int i = 0; int noOfCreated, noOfCreatedErrors; foreach (Proxy.NorsService.WebSpeciesObservation norsSpeciesObservation in webSpeciesObservationChange.CreatedSpeciesObservations) { WebData webData = new WebData(); webData.DataFields = new List <WebDataField>(); foreach (WebSpeciesObservationField webSpeciesObservationField in norsSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.Information = webSpeciesObservationField.Information; webDataField.Name = webSpeciesObservationField.Property.Id.ToString(); webDataField.Type = (WebDataType)webSpeciesObservationField.Type; webDataField.Unit = webSpeciesObservationField.Unit; webDataField.Value = webSpeciesObservationField.Value.CheckInjection(); if (webDataField.IsNotNull()) { webData.DataFields.Add(webDataField); ////Debug.Write(webDataField.Name + " : "); ////Debug.Write(webDataField.Value + " : "); ////Debug.Write(webDataField.Type + " : "); ////Debug.Write(webDataField.Unit + " : "); ////Debug.WriteLine(webDataField.Information); } } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = norsProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); // UPDATED i = 0; int noOfUpdated, noOfUpdatedErrors; foreach (Proxy.NorsService.WebSpeciesObservation norsSpeciesObservation in webSpeciesObservationChange.UpdatedSpeciesObservations) { WebData webData = new WebData(); webData.DataFields = new List <WebDataField>(); foreach (WebSpeciesObservationField webSpeciesObservationField in norsSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.Information = webSpeciesObservationField.Information; webDataField.Name = webSpeciesObservationField.Property.Id.ToString(); webDataField.Type = (WebDataType)webSpeciesObservationField.Type; webDataField.Unit = webSpeciesObservationField.Unit; webDataField.Value = webSpeciesObservationField.Value.CheckInjection(); if (webDataField.IsNotNull()) { webData.DataFields.Add(webDataField); } } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = norsProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.UpdatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfUpdated; sumNoOfCreatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); // DELETED i = 0; int noOfDeleted = 0, noOfDeletedErrors = 0; speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); foreach (String norsSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) { // if (adsSpeciesObservation.DatabaseId == 99) continue; String id = norsSpeciesObservation.Substring(norsSpeciesObservation.LastIndexOf('.') + 1); speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } // write remaining observations to database connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); // HANDLE LOOP OVER CHANGEID webSpeciesObservationChange = WebServiceProxy.NorsService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, webSpeciesObservationChange.MaxChangeId + 1, isChangeIdspecified, maxReturnedChanges); if (decimal.Remainder(++j, 10) != 0) { continue; } Debug.WriteLine(webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); } Debug.WriteLine("end :" + webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); // Log latest harvest date for the data provider // Use changeFrom since changedTo is calculated (+1 day) context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedFrom); context.GetSpeciesObservationDatabase() .LogHarvestRead( context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors, currentChangeId); stopwatch.Stop(); }
/// <summary> /// The process observation. /// </summary> /// <param name="webData"> /// The web data. /// </param> /// <param name="mappings"> /// The mappings. /// </param> /// <param name="context"> /// The context. /// </param> /// <returns> /// The harvestSpeciesObservation.<see cref="HarvestSpeciesObservation"/>. /// </returns> /// <exception cref="ArgumentNullException">If some properties where not found. /// </exception> public virtual HarvestSpeciesObservation ProcessObservation(WebData webData, List <HarvestMapping> mappings, WebServiceContext context) { // Create a dictionary object from the WebData object (Not cachable, needs to be done for each row) // Can be a problem if same name exists in more than one class... Dictionary <string, WebDataField> dictionaryWebData = webData.DataFields.ToDictionary(row => row.Name.ToLower(), row => row); WebLocale webLocale = GetWebLocale(); HarvestSpeciesObservation harvestSpeciesObservation = new HarvestSpeciesObservation(); harvestSpeciesObservation.Fields = new List <HarvestSpeciesObservationField>(); if (mappings.IsNotNull()) { // Loop through the mappings + create objects for every field foreach (HarvestMapping mapping in mappings) { String mapProperty; if (!ReadElement(mapping.Property, out mapProperty)) { throw new ArgumentNullException(String.Format("Error in mapping. Element <Property>, {0}, not found.", mapping.Property)); } SpeciesObservationPropertyId speciesObservationPropertyId = SpeciesObservationPropertyId.None; // This validation should not be performed on fields of ProjectParameter type if (mapping.GUID == null || !mapping.GUID.Contains("projectparameter")) { if (!Enum.TryParse(mapProperty, true, out speciesObservationPropertyId)) { throw new ArgumentNullException(String.Format("Property value: {0} not a valid property.", mapProperty)); } } Boolean isMandatory; String isMandatoryString; if (!ReadElement(mapping.Mandatory.ToString(), out isMandatoryString)) { throw new ArgumentNullException(String.Format("Error in mapping. Element <IsMandatoryFromProvider>, {0}, not found.", mapping.IsMandatoryFromProvider)); } isMandatory = mapping.Mandatory; String mapClass; if (!ReadElement(mapping.Class, out mapClass)) { throw new ArgumentNullException(String.Format("Error in mapping. Element <Class>, {0}, not found.", mapping.Class)); } SpeciesObservationClassId speciesObservationClassId; if (!Enum.TryParse(mapClass, true, out speciesObservationClassId)) { throw new ArgumentNullException(String.Format("Class value: {0} not a valid class.", mapClass)); } String mapType; if (!ReadElement(mapping.Type, out mapType)) { throw new ArgumentNullException(String.Format("Error in mapping. Element <Type>, {0}, not found.", mapping.Type)); } WebDataType webDataType; if (!Enum.TryParse(mapType, true, out webDataType)) { throw new ArgumentNullException(String.Format("Type value: {0} not a valid type.", mapType)); } String mapName; if (mapping.IsProjectParameter()) { mapName = mapping.PropertyIdentifier; } else { ReadElement(mapping.Name, out mapName); } String mapMethod; ReadElement(mapping.Method, out mapMethod); String mapDefault; ReadElement(mapping.Default, out mapDefault); String fieldValue = String.Empty; String fieldUnit = null; if (mapName.IsNotEmpty()) { try { if (dictionaryWebData.ContainsKey(mapName.ToLower())) { fieldValue = dictionaryWebData[mapName.ToLower()].Value; fieldUnit = ReplaceInvalidCharacters(dictionaryWebData[mapName.ToLower()].Unit); if (webDataType == WebDataType.String) { fieldValue = ReplaceInvalidCharacters(fieldValue); } } } catch (KeyNotFoundException) { throw new ArgumentNullException(String.Format("Field:{0} not found.", mapName)); } } if (mapMethod.IsNotEmpty()) { fieldValue = MethodWrapper(mapMethod, dictionaryWebData, context, mapping); } if (fieldValue.IsNull() || fieldValue.IsEmpty()) { // field is empty - set to default value ? if (mapDefault.IsNotEmpty()) { fieldValue = mapDefault; } } if (fieldValue.IsNotEmpty() || isMandatory) { HarvestSpeciesObservationField harvestSpeciesObservationField = new HarvestSpeciesObservationField(); harvestSpeciesObservationField.Class = new WebSpeciesObservationClass(speciesObservationClassId); // The type SpeciesObservationPropertyId.None indicates the use of the Identifier instead harvestSpeciesObservationField.Property = new WebSpeciesObservationProperty(); if (mapping.IsProjectParameter()) { harvestSpeciesObservationField.Property.Id = SpeciesObservationPropertyId.None; harvestSpeciesObservationField.Property.Identifier = mapping.PropertyIdentifier; } else { harvestSpeciesObservationField.Property.Id = speciesObservationPropertyId; if (speciesObservationPropertyId == SpeciesObservationPropertyId.None) { harvestSpeciesObservationField.Property.Identifier = mapping.Name; } } harvestSpeciesObservationField.Type = webDataType; harvestSpeciesObservationField.Unit = fieldUnit; harvestSpeciesObservationField.Value = fieldValue.CheckInjection(); harvestSpeciesObservationField.Locale = webLocale; harvestSpeciesObservationField.IsDarwinCore = mapping.IsDarwinCore; harvestSpeciesObservationField.IsSearchable = mapping.IsSearchable; harvestSpeciesObservationField.IsMandatory = isMandatory; // måste med till nästa steg harvestSpeciesObservationField.IsMandatoryFromProvider = mapping.IsMandatoryFromProvider; harvestSpeciesObservationField.IsObtainedFromProvider = mapping.IsObtainedFromProvider; harvestSpeciesObservationField.PersistedInTable = mapping.PersistedInTable; // add the field to the observation harvestSpeciesObservation.Fields.Add(harvestSpeciesObservationField); } } } // return one observation return(harvestSpeciesObservation); }
/// <summary> /// Get information about species observations /// that has it's id in the SightingIds list. /// </summary> /// <param name="sightingIds">A list of id's to return information about.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> public void GetSpeciesObservationChange(List <string> sightingIds, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebData webData; SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; using (ArtportalenServer artportalenServer = new ArtportalenServer()) { ArtportalenProcess artportalenProcess = new ArtportalenProcess(); // Get created and edited observations from Artportalen using (DataReader dataReader = artportalenServer.GetSpeciesObservationsByIds(sightingIds)) { speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); int i = 0; int noOfCreated, noOfCreatedErrors; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); HarvestSpeciesObservation harvestSpeciesObservation = artportalenProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems. connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); speciesObservationChange.UpdatedSpeciesObservations.Clear(); } } // Write the remaining observations to database connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); } //// Get deleted observations from Artportalen //using (DataReader dataReader = artportalenServer.GetDeletedObservations(changedFrom, calcChangedTo)) //{ // speciesObservationChange.DeletedSpeciesObservationGuids = new List<String>(); // int i = 0; // while (dataReader.Read()) // { // webData = new WebData(); // webData.LoadData(dataReader); // String deletedCatalogNumber = artportalenProcess.ProcessDeletedObservation(webData); // speciesObservationChange.DeletedSpeciesObservationGuids.Add(deletedCatalogNumber); // if (decimal.Remainder(++i, 10000) == 0) // { // // Write every 10000 observation to database to avoid memory problems // connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); // sumNoOfDeleted += noOfDeleted; // sumNoOfDeletedErrors += noOfDeletedErrors; // speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); // } // } // // Write the remaining observations to database // connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); // sumNoOfDeleted += noOfDeleted; // sumNoOfDeletedErrors += noOfDeletedErrors; // speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); //} } // Log latest harvest date for the data provider //context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedTo); context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, new DateTime(1900, 01, 01), new DateTime(1900, 01, 01), stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors); stopwatch.Stop(); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { DateTime calcChangedTo = changedTo.AddDays(1); calcChangedTo = new DateTime(calcChangedTo.Year, calcChangedTo.Month, calcChangedTo.Day); WebData webData; SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebSpeciesObservationClass speciesObservationClass = new WebSpeciesObservationClass(SpeciesObservationClassId.DarwinCore); WebSpeciesObservationProperty reportedDateProp = new WebSpeciesObservationProperty(SpeciesObservationPropertyId.ReportedDate); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; using (ObservationsdatabasenServer observationDatabasenServer = new ObservationsdatabasenServer()) { ObservationsdatabasenProcess observationsdatabasenProcess = new ObservationsdatabasenProcess(); // Get created and edited observations from Observationsdatabasen using (DataReader dataReader = observationDatabasenServer.GetSpeciesObservations(changedFrom, calcChangedTo)) { speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); speciesObservationChange.UpdatedSpeciesObservations = new List <HarvestSpeciesObservation>(); int i = 0; int noOfCreated, noOfCreatedErrors, noOfUpdated, noOfUpdatedErrors; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); HarvestSpeciesObservation harvestSpeciesObservation = observationsdatabasenProcess.ProcessObservation(webData, mappings, context); DateTime reportedDate = harvestSpeciesObservation.GetFieldValue(speciesObservationClass, reportedDateProp).WebParseDateTime(); // If reportedDate is earlier than changedFrom the observation is edited. if (reportedDate < changedFrom) { speciesObservationChange.UpdatedSpeciesObservations.Add(harvestSpeciesObservation); } else { speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); } if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); speciesObservationChange.UpdatedSpeciesObservations.Clear(); } } // Write the remaining observations to database connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); speciesObservationChange.UpdatedSpeciesObservations.Clear(); } // Get deleted observations from Observationsdatabasen using (DataReader dataReader = observationDatabasenServer.GetDeletedObservations(changedFrom, calcChangedTo)) { speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); int noOfDeleted, noOfDeletedErrors; int i = 0; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); String deletedCatalogNumber = observationsdatabasenProcess.ProcessDeletedObservation(webData); speciesObservationChange.DeletedSpeciesObservationGuids.Add(deletedCatalogNumber); if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } // Log latest harvest date for the data provider context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedTo); context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors); stopwatch.Stop(); return(false); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range or /// from a specified changeId. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom"> /// Changed from date. /// </param> /// <param name="isChangedFromSpecified"> /// Is changed from specified. /// </param> /// <param name="changedTo"> /// Changed to date. /// </param> /// <param name="isChangedToSpecified"> /// Is changed to specified. /// </param> /// <param name="changeId"> /// From witch change id. /// </param> /// <param name="isChangeIdspecified"> /// Is changed id specified. /// </param> /// <param name="maxReturnedChanges"> /// Max number of observations returned. /// </param> /// <param name="mappings"> /// The mapping list. /// </param> /// <param name="context"> /// The web service context. /// </param> /// <param name="connectorServer"> /// The connector service. /// </param> public void GetSpeciesObservationChange(DateTime changedFrom, Boolean isChangedFromSpecified, DateTime changedTo, Boolean isChangedToSpecified, Int64 changeId, Boolean isChangeIdspecified, Int64 maxReturnedChanges, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange { CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(), UpdatedSpeciesObservations = new List <HarvestSpeciesObservation>() }; var stopwatch = Stopwatch.StartNew(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; KulProcess kulProcess = new KulProcess(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); Stopwatch stopwatch1 = Stopwatch.StartNew(); WebSpeciesObservationChange webSpeciesObservationChange = WebServiceProxy.KulService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, changeId, isChangeIdspecified, maxReturnedChanges); Debug.WriteLine("Harvest: " + DateTime.Now.ToLongTimeString() + " - Time: " + TimeSpan.FromMilliseconds(stopwatch1.ElapsedMilliseconds).TotalMinutes + " Minutes"); // LOOP OVER CHANGEID // Denna verkar inte klara mer än ca 100 obsar så det behövs en loop över dessa... Int32 readSize = 0; Int64 currentChangeId = 0; int j = 0; while ((readSize++ < 50) && (currentChangeId < webSpeciesObservationChange.MaxChangeId)) { currentChangeId = webSpeciesObservationChange.MaxChangeId; context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, webSpeciesObservationChange.MaxChangeId); // CREATED AddNewSpeciesObservations(webSpeciesObservationChange, speciesObservationChange, kulProcess, mappings, context, connectorServer, dataProvider, out sumNoOfCreated, out sumNoOfCreatedErrors); /* int i = 0; * int noOfCreated, noOfCreatedErrors; * foreach (Proxy.KulService.WebSpeciesObservation kulSpeciesObservation in webSpeciesObservationChange.CreatedSpeciesObservations) * { * WebData webData = new WebData { DataFields = new List<WebDataField>() }; * * foreach (WebSpeciesObservationField webSpeciesObservationField in kulSpeciesObservation.Fields) * { * WebDataField webDataField = new WebDataField * { * Information = webSpeciesObservationField.Information, * Name = webSpeciesObservationField.Property.Id.ToString(), * Type = (WebDataType)webSpeciesObservationField.Type, * Unit = webSpeciesObservationField.Unit, * Value = webSpeciesObservationField.Value.CheckInjection() * }; * * if (webDataField.IsNotNull()) * { * webData.DataFields.Add(webDataField); * } * } * * // map webdata * HarvestSpeciesObservation harvestSpeciesObservation = kulProcess.ProcessObservation(webData, mappings, context); * * speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); * * if (decimal.Remainder(++i, 1000) != 0) * { * continue; * } * * // write every 10000 observation to database to avoid memory problems * connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); * * sumNoOfCreated += noOfCreated; * sumNoOfCreatedErrors += noOfCreatedErrors; * speciesObservationChange.CreatedSpeciesObservations.Clear(); * } * * connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); * * sumNoOfCreated += noOfCreated; * sumNoOfCreatedErrors += noOfCreatedErrors; * speciesObservationChange.CreatedSpeciesObservations.Clear(); */ // UPDATED int i = 0; int noOfUpdated, noOfUpdatedErrors; foreach (Proxy.KulService.WebSpeciesObservation kulSpeciesObservation in webSpeciesObservationChange.UpdatedSpeciesObservations) { WebData webData = new WebData { DataFields = new List <WebDataField>() }; foreach (WebSpeciesObservationField webSpeciesObservationField in kulSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField { Information = webSpeciesObservationField.Information, Name = webSpeciesObservationField.Property.Id.ToString(), Type = (WebDataType)webSpeciesObservationField.Type, Unit = webSpeciesObservationField.Unit, Value = webSpeciesObservationField.Value.CheckInjection() }; if (webDataField.IsNotNull()) { webData.DataFields.Add(webDataField); } } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = kulProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.UpdatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 1000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfUpdated; sumNoOfCreatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); // DELETED this.RemoveDeletedSpeciesObservations(webSpeciesObservationChange, speciesObservationChange, kulProcess, mappings, context, connectorServer, dataProvider, out sumNoOfDeleted, out sumNoOfDeletedErrors); /* * i = 0; * int noOfDeleted = 0, noOfDeletedErrors = 0; * speciesObservationChange.DeletedSpeciesObservationGuids = new List<String>(); * * foreach (String kulSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) * { * // if (adsSpeciesObservation.DatabaseId == 99) continue; * String id = kulSpeciesObservation.Substring(kulSpeciesObservation.LastIndexOf('.') + 1); * speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); * * if (decimal.Remainder(++i, 1000) != 0) * { * continue; * } * * // write every 10000 observation to database to avoid memory problems * connectorServer.DeleteSpeciesObservations( * context, * speciesObservationChange.DeletedSpeciesObservationGuids, * dataProvider, * out noOfDeleted, * out noOfDeletedErrors); * sumNoOfDeleted += noOfDeleted; * sumNoOfDeletedErrors += noOfDeletedErrors; * speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); * } * * // write remaining observations to database * connectorServer.DeleteSpeciesObservations( * context, * speciesObservationChange.DeletedSpeciesObservationGuids, * dataProvider, * out noOfDeleted, * out noOfDeletedErrors); * sumNoOfDeleted += noOfDeleted; * sumNoOfDeletedErrors += noOfDeletedErrors; * speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); */ // HANDLE LOOP OVER CHANGEID webSpeciesObservationChange = WebServiceProxy.KulService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, webSpeciesObservationChange.MaxChangeId + 1, isChangeIdspecified, maxReturnedChanges); if (decimal.Remainder(++j, 10) != 0) { continue; } Debug.WriteLine(webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); } Debug.WriteLine("end :" + webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); // Log latest harvest date for the data provider // Use changeFrom since changedTo is calculated (+1 day) context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedFrom); context.GetSpeciesObservationDatabase() .LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors, currentChangeId); stopwatch.Stop(); }
/// <summary> The add new species observations. </summary> /// <param name="webSpeciesObservationChange"> The web species observation change. </param> /// <param name="speciesObservationChange"> The species observation change. </param> /// <param name="kulProcess"> The kul Process. </param> /// <param name="mappings"> The mappings. </param> /// <param name="context"> The context. </param> /// <param name="connectorServer"> The connector Server. </param> /// <param name="dataProvider"> The data Provider. </param> /// <param name="sumNoOfCreated"> The sum No Of Created. </param> /// <param name="sumNoOfCreatedErrors"> The sum No Of Created Errors. </param> private void AddNewSpeciesObservations( WebSpeciesObservationChange webSpeciesObservationChange, SpeciesObservationChange speciesObservationChange, KulProcess kulProcess, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer, WebSpeciesObservationDataProvider dataProvider, out int sumNoOfCreated, out int sumNoOfCreatedErrors) { int observationCount = 0; int noOfCreated, noOfCreatedErrors; sumNoOfCreated = 0; sumNoOfCreatedErrors = 0; foreach (Proxy.KulService.WebSpeciesObservation kulSpeciesObservation in webSpeciesObservationChange.CreatedSpeciesObservations) { WebData webData = new WebData { DataFields = new List <WebDataField>() }; foreach (WebSpeciesObservationField webSpeciesObservationField in kulSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField { Information = webSpeciesObservationField.Information, Name = webSpeciesObservationField.Property.Id.ToString(), Type = (WebDataType)webSpeciesObservationField.Type, Unit = webSpeciesObservationField.Unit, Value = webSpeciesObservationField.Value.CheckInjection() }; webData.DataFields.Add(webDataField); } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = kulProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++observationCount, 1000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); }