/// <summary> The remove deleted species observations. </summary> /// <param name="webSpeciesObservationChange"> The web species observation change. </param> /// <param name="speciesObservationChange"> The species observation change. </param> /// <param name="kulProcess"> The kul process. </param> /// <param name="mappings"> The mappings. </param> /// <param name="context"> The context. </param> /// <param name="connectorServer"> The connector server. </param> /// <param name="dataProvider"> The data provider. </param> /// <param name="sumNoOfDeleted"> The sum No Of Deleted. </param> /// <param name="sumNoOfDeletedErrors"> The sum No Of Deleted Errors. </param> private void RemoveDeletedSpeciesObservations( WebSpeciesObservationChange webSpeciesObservationChange, SpeciesObservationChange speciesObservationChange, KulProcess kulProcess, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer, WebSpeciesObservationDataProvider dataProvider, out int sumNoOfDeleted, out int sumNoOfDeletedErrors) { sumNoOfDeleted = 0; sumNoOfDeletedErrors = 0; int observationCount = 0; int noOfDeleted = 0, noOfDeletedErrors = 0; speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); foreach (String kulSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) { // if (adsSpeciesObservation.DatabaseId == 99) continue; String id = kulSpeciesObservation.Substring(kulSpeciesObservation.LastIndexOf('.') + 1); speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); if (decimal.Remainder(++observationCount, 1000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } // write remaining observations to database connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range or /// from a specified changeId. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom"> /// Changed from date. /// </param> /// <param name="isChangedFromSpecified"> /// Is changed from specified. /// </param> /// <param name="changedTo"> /// Changed to date. /// </param> /// <param name="isChangedToSpecified"> /// Is changed to specified. /// </param> /// <param name="changeId"> /// From witch change id. /// </param> /// <param name="isChangeIdspecified"> /// Is changed id specified. /// </param> /// <param name="maxReturnedChanges"> /// Max number of observations returned. /// </param> /// <param name="mappings"> /// The mapping list. /// </param> /// <param name="context"> /// The web service context. /// </param> /// <param name="connectorServer"> /// The connector service. /// </param> public void GetSpeciesObservationChange(DateTime changedFrom, Boolean isChangedFromSpecified, DateTime changedTo, Boolean isChangedToSpecified, Int64 changeId, Boolean isChangeIdspecified, Int64 maxReturnedChanges, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); speciesObservationChange.UpdatedSpeciesObservations = new List <HarvestSpeciesObservation>(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; NorsProcess norsProcess = new NorsProcess(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebSpeciesObservationChange webSpeciesObservationChange = WebServiceProxy.NorsService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, changeId, isChangeIdspecified, maxReturnedChanges); // LOOP OVER CHANGEID // Denna verkar inte klara mer än ca 100 obsar så det behövs en loop över dessa... Int32 readSize = 0; Int64 currentChangeId = 0; int j = 0; while ((readSize++ < 50) && (currentChangeId < webSpeciesObservationChange.MaxChangeId)) { currentChangeId = webSpeciesObservationChange.MaxChangeId; context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, webSpeciesObservationChange.MaxChangeId); // CREATED int i = 0; int noOfCreated, noOfCreatedErrors; foreach (Proxy.NorsService.WebSpeciesObservation norsSpeciesObservation in webSpeciesObservationChange.CreatedSpeciesObservations) { WebData webData = new WebData(); webData.DataFields = new List <WebDataField>(); foreach (WebSpeciesObservationField webSpeciesObservationField in norsSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.Information = webSpeciesObservationField.Information; webDataField.Name = webSpeciesObservationField.Property.Id.ToString(); webDataField.Type = (WebDataType)webSpeciesObservationField.Type; webDataField.Unit = webSpeciesObservationField.Unit; webDataField.Value = webSpeciesObservationField.Value.CheckInjection(); if (webDataField.IsNotNull()) { webData.DataFields.Add(webDataField); ////Debug.Write(webDataField.Name + " : "); ////Debug.Write(webDataField.Value + " : "); ////Debug.Write(webDataField.Type + " : "); ////Debug.Write(webDataField.Unit + " : "); ////Debug.WriteLine(webDataField.Information); } } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = norsProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); // UPDATED i = 0; int noOfUpdated, noOfUpdatedErrors; foreach (Proxy.NorsService.WebSpeciesObservation norsSpeciesObservation in webSpeciesObservationChange.UpdatedSpeciesObservations) { WebData webData = new WebData(); webData.DataFields = new List <WebDataField>(); foreach (WebSpeciesObservationField webSpeciesObservationField in norsSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.Information = webSpeciesObservationField.Information; webDataField.Name = webSpeciesObservationField.Property.Id.ToString(); webDataField.Type = (WebDataType)webSpeciesObservationField.Type; webDataField.Unit = webSpeciesObservationField.Unit; webDataField.Value = webSpeciesObservationField.Value.CheckInjection(); if (webDataField.IsNotNull()) { webData.DataFields.Add(webDataField); } } // map webdata HarvestSpeciesObservation harvestSpeciesObservation = norsProcess.ProcessObservation(webData, mappings, context); speciesObservationChange.UpdatedSpeciesObservations.Add(harvestSpeciesObservation); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfUpdated; sumNoOfCreatedErrors += noOfUpdatedErrors; speciesObservationChange.UpdatedSpeciesObservations.Clear(); // DELETED i = 0; int noOfDeleted = 0, noOfDeletedErrors = 0; speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); foreach (String norsSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) { // if (adsSpeciesObservation.DatabaseId == 99) continue; String id = norsSpeciesObservation.Substring(norsSpeciesObservation.LastIndexOf('.') + 1); speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); if (decimal.Remainder(++i, 10000) != 0) { continue; } // write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } // write remaining observations to database connectorServer.DeleteSpeciesObservations( context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); // HANDLE LOOP OVER CHANGEID webSpeciesObservationChange = WebServiceProxy.NorsService.GetSpeciesObservationChangeAsSpecies(changedFrom, isChangedFromSpecified, changedTo, isChangedToSpecified, webSpeciesObservationChange.MaxChangeId + 1, isChangeIdspecified, maxReturnedChanges); if (decimal.Remainder(++j, 10) != 0) { continue; } Debug.WriteLine(webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); } Debug.WriteLine("end :" + webSpeciesObservationChange.MaxChangeId + " : count created: " + sumNoOfCreated); // Log latest harvest date for the data provider // Use changeFrom since changedTo is calculated (+1 day) context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedFrom); context.GetSpeciesObservationDatabase() .LogHarvestRead( context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors, currentChangeId); stopwatch.Stop(); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { Int64 currentPage; Int32 noOfCreated, noOfCreatedErrors; Int32 noOfDeleted, noOfDeletedErrors; Int64 nextPage, pageCount, speciesObservationCount; Tuple <List <HarvestSpeciesObservation>, List <String> > page; Stopwatch stopwatch; WebSpeciesObservationDataProvider dataProvider; stopwatch = new Stopwatch(); stopwatch.Start(); dataProvider = GetSpeciesObservationDataProvider(context); currentPage = context.GetSpeciesObservationDatabase().GetMaxChangeId(dataProvider.Id); speciesObservationCount = GetSpeciesObservationCount(); pageCount = GetPageCount(speciesObservationCount); if (currentPage <= pageCount) { nextPage = currentPage; if (nextPage < pageCount) { nextPage++; } // Get species observation information. page = GetSpeciesObservationPage(mappings, context, nextPage, MaxSpeciesObservationsPerPage); // New species observations. connectorServer.UpdateSpeciesObservations(context, page.Item1, dataProvider, out noOfCreated, out noOfCreatedErrors); // Deleted species observations. connectorServer.DeleteSpeciesObservations(context, page.Item2, dataProvider, out noOfDeleted, out noOfDeletedErrors); context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, noOfCreated, noOfCreatedErrors, -1, -1, noOfDeleted, noOfDeletedErrors, currentPage); stopwatch.Stop(); Debug.WriteLine(string.Format("SHARK handled page {0} of {1} in {2} msec", nextPage, pageCount, stopwatch.ElapsedMilliseconds)); if (nextPage <= pageCount) { context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, nextPage); } // Log latest harvest date for the data provider context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedTo); } return(false); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { Int32 noOfDeleted, noOfDeletedErrors; int noOfUpdated = 0, noOfUpdatedErrors = 0; DateTime calcChangedTo = changedTo.AddDays(1); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebData webData; Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); if (mUpdatedSpeciesObservations.IsEmpty() && mUpdatedSpeciesObservationIds.IsEmpty()) { SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); mUpdatedSpeciesObservationIds = new List <Int64>(); mUpdatedSpeciesObservations = new List <WebData>(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; using (ArtportalenServer artportalenServer = new ArtportalenServer()) { var artportalenProcess = new ArtportalenProcess(); // Get changes from Artportalen. using (DataReader dataReader = artportalenServer.GetSpeciesObservations(changedFrom, calcChangedTo)) { while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); if (mUpdatedSpeciesObservations.Count >= 100000) { mUpdatedSpeciesObservationIds.Add(GetSpeciesObservationId(webData)); } else { mUpdatedSpeciesObservations.Add(webData); } } AddProjectParameters(this.mUpdatedSpeciesObservations, dataReader); } // Save updated species observations into database. // Max 10000 species observations are saved in one call. UpdateSpeciesObservations(context, mUpdatedSpeciesObservations, dataProvider, connectorServer, mappings, out noOfUpdated, out noOfUpdatedErrors); sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; // Get deleted observations from Artportalen speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); using (DataReader dataReader = artportalenServer.GetDeletedObservations(changedFrom, calcChangedTo)) { int i = 0; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); String deletedCatalogNumber = artportalenProcess.ProcessDeletedObservation(webData); speciesObservationChange.DeletedSpeciesObservationGuids.Add(deletedCatalogNumber); if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } // Write the remaining observations to database connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } // Log latest harvest date for the data provider context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedTo); context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors); } else { // Save updated species observations into database. // Max 10000 species observations are saved in one call. if (mUpdatedSpeciesObservations.IsNotEmpty()) { UpdateSpeciesObservations(context, mUpdatedSpeciesObservations, dataProvider, connectorServer, mappings, out noOfUpdated, out noOfUpdatedErrors); } else { if (mUpdatedSpeciesObservationIds.IsNotEmpty()) { UpdateSpeciesObservations(context, mUpdatedSpeciesObservationIds, dataProvider, connectorServer, mappings, out noOfUpdated, out noOfUpdatedErrors); } } context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, 0, 0, noOfUpdated, noOfUpdatedErrors, 0, 0); } stopwatch.Stop(); return(mUpdatedSpeciesObservations.IsNotEmpty() || mUpdatedSpeciesObservationIds.IsNotEmpty()); }
/// <summary> /// Get information about species observations that has changed. /// The service contains observations that are aggregated and has start /// and end dates that are set to the first and last day of every month. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { Dictionary <String, WebSpeciesObservation> updatedSpeciesObservations; SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); WramProcess wramProcess = new WramProcess(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebSpeciesObservationChange webSpeciesObservationChange = WebServiceProxy.WramService.GetSpeciesObservationChangeAsSpecies(DateTime.Now, false, DateTime.Now, false, dataProvider.MaxChangeId, true, Settings.Default.MaxSpeciesObservationsFromWram); // Run all created and updated together as one list. updatedSpeciesObservations = new Dictionary <String, WebSpeciesObservation>(); AddSpeciesObservations(updatedSpeciesObservations, webSpeciesObservationChange.UpdatedSpeciesObservations); AddSpeciesObservations(updatedSpeciesObservations, webSpeciesObservationChange.CreatedSpeciesObservations); List <WebSpeciesObservation> createdAndUpdatedObservations = new List <WebSpeciesObservation>(); createdAndUpdatedObservations.AddRange(updatedSpeciesObservations.Values); // Handle created and updated species observations. int noOfCreated = 0, noOfCreatedErrors = 0; if (createdAndUpdatedObservations.IsNotEmpty()) { foreach (WebSpeciesObservation wramSpeciesObservation in createdAndUpdatedObservations) { WebData webData = new WebData { DataFields = new List <WebDataField>() }; foreach (WebSpeciesObservationField webSpeciesObservationField in wramSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.LoadData(webSpeciesObservationField); webData.DataFields.Add(webDataField); } // Map and add webdata to the 'created collection' speciesObservationChange.CreatedSpeciesObservations.Add(wramProcess.ProcessObservation(webData, mappings, context)); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); } // Handle deleted species observtions. int noOfDeleted = 0, noOfDeletedErrors = 0; speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); if (webSpeciesObservationChange.DeletedSpeciesObservationGuids.IsNotEmpty()) { foreach (String wramSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) { String id = wramSpeciesObservation.Substring(wramSpeciesObservation.LastIndexOf(':') + 1); speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); } connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); } Debug.WriteLine(webSpeciesObservationChange.MaxChangeId + " : count created: " + noOfCreated); context.GetSpeciesObservationDatabase().LogHarvestRead( context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, noOfCreated, noOfCreatedErrors, 0, 0, noOfDeleted, noOfDeletedErrors, webSpeciesObservationChange.MaxChangeId); stopwatch.Stop(); if (webSpeciesObservationChange.MaxChangeId > 0) { // context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, webSpeciesObservationChange.MaxChangeId); // Testing if WRAM has a problem with handling of change id. context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, webSpeciesObservationChange.MaxChangeId - 1); } // else: No changes was retrieved. There are no more changes available right now. return(false); }
/// <summary> /// Get information about species observations /// that has changed in the specified date range. /// Only date part of parameters changedFrom and changedTo /// are used. It does not matter what time of day that is set /// in parameters changedFrom and changedTo. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { DateTime calcChangedTo = changedTo.AddDays(1); calcChangedTo = new DateTime(calcChangedTo.Year, calcChangedTo.Month, calcChangedTo.Day); WebData webData; SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); WebSpeciesObservationClass speciesObservationClass = new WebSpeciesObservationClass(SpeciesObservationClassId.DarwinCore); WebSpeciesObservationProperty reportedDateProp = new WebSpeciesObservationProperty(SpeciesObservationPropertyId.ReportedDate); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); int sumNoOfCreated = 0, sumNoOfCreatedErrors = 0, sumNoOfUpdated = 0, sumNoOfUpdatedErrors = 0, sumNoOfDeleted = 0, sumNoOfDeletedErrors = 0; using (ObservationsdatabasenServer observationDatabasenServer = new ObservationsdatabasenServer()) { ObservationsdatabasenProcess observationsdatabasenProcess = new ObservationsdatabasenProcess(); // Get created and edited observations from Observationsdatabasen using (DataReader dataReader = observationDatabasenServer.GetSpeciesObservations(changedFrom, calcChangedTo)) { speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); speciesObservationChange.UpdatedSpeciesObservations = new List <HarvestSpeciesObservation>(); int i = 0; int noOfCreated, noOfCreatedErrors, noOfUpdated, noOfUpdatedErrors; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); HarvestSpeciesObservation harvestSpeciesObservation = observationsdatabasenProcess.ProcessObservation(webData, mappings, context); DateTime reportedDate = harvestSpeciesObservation.GetFieldValue(speciesObservationClass, reportedDateProp).WebParseDateTime(); // If reportedDate is earlier than changedFrom the observation is edited. if (reportedDate < changedFrom) { speciesObservationChange.UpdatedSpeciesObservations.Add(harvestSpeciesObservation); } else { speciesObservationChange.CreatedSpeciesObservations.Add(harvestSpeciesObservation); } if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); speciesObservationChange.UpdatedSpeciesObservations.Clear(); } } // Write the remaining observations to database connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.UpdatedSpeciesObservations, dataProvider, out noOfUpdated, out noOfUpdatedErrors); sumNoOfCreated += noOfCreated; sumNoOfCreatedErrors += noOfCreatedErrors; sumNoOfUpdated += noOfUpdated; sumNoOfUpdatedErrors += noOfUpdatedErrors; speciesObservationChange.CreatedSpeciesObservations.Clear(); speciesObservationChange.UpdatedSpeciesObservations.Clear(); } // Get deleted observations from Observationsdatabasen using (DataReader dataReader = observationDatabasenServer.GetDeletedObservations(changedFrom, calcChangedTo)) { speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); int noOfDeleted, noOfDeletedErrors; int i = 0; while (dataReader.Read()) { webData = new WebData(); webData.LoadData(dataReader); String deletedCatalogNumber = observationsdatabasenProcess.ProcessDeletedObservation(webData); speciesObservationChange.DeletedSpeciesObservationGuids.Add(deletedCatalogNumber); if (decimal.Remainder(++i, 10000) == 0) { // Write every 10000 observation to database to avoid memory problems connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); sumNoOfDeleted += noOfDeleted; sumNoOfDeletedErrors += noOfDeletedErrors; speciesObservationChange.DeletedSpeciesObservationGuids.Clear(); } } // Log latest harvest date for the data provider context.GetSpeciesObservationDatabase().SetDataProviderLatestHarvestDate(dataProvider.Id, changedTo); context.GetSpeciesObservationDatabase().LogHarvestRead(context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, sumNoOfCreated, sumNoOfCreatedErrors, sumNoOfUpdated, sumNoOfUpdatedErrors, sumNoOfDeleted, sumNoOfDeletedErrors); stopwatch.Stop(); return(false); }
/// <summary> /// Get information about species observations that has changed. /// The service contains observations that are aggregated and has start /// and end dates that are set to the first and last day of every month. /// </summary> /// <param name="changedFrom">Start date for changes.</param> /// <param name="changedTo">End date for changes.</param> /// <param name="mappings">The mappings.</param> /// <param name="context">Web service context.</param> /// <param name="connectorServer">The connector server.</param> /// <returns> /// Returns true if there are more species /// observations to retrieve for current date. /// </returns> public Boolean GetSpeciesObservationChange(DateTime changedFrom, DateTime changedTo, List <HarvestMapping> mappings, WebServiceContext context, IConnectorServer connectorServer) { Dictionary <String, WebSpeciesObservation> updatedSpeciesObservations; Int32 minutes; Int64 maxChangeId; // Wait for MVM service to be ready for use. for (minutes = 0; minutes < 60; minutes++) { if (WebServiceProxy.MvmService.IsReadyToUse()) { break; } else { // Wait one minute for MVM service to be ready for use. Thread.Sleep(60000); } } if ((minutes == 60) && !(WebServiceProxy.MvmService.IsReadyToUse())) { // Can not wait any longer for MVM service to be ready. return(false); } SpeciesObservationChange speciesObservationChange = new SpeciesObservationChange(); speciesObservationChange.CreatedSpeciesObservations = new List <HarvestSpeciesObservation>(); WebSpeciesObservationChange webSpeciesObservationChange; webSpeciesObservationChange = new WebSpeciesObservationChange(); webSpeciesObservationChange.CreatedSpeciesObservations = new List <WebSpeciesObservation>(); webSpeciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); webSpeciesObservationChange.UpdatedSpeciesObservations = new List <WebSpeciesObservation>(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); MvmProcess mvmProcess = new MvmProcess(); WebSpeciesObservationDataProvider dataProvider = GetSpeciesObservationDataProvider(context); maxChangeId = dataProvider.MaxChangeId; for (Int32 index = 0; index < 10; index++) { WebSpeciesObservationChange webSpeciesObservationChangeTemp = WebServiceProxy.MvmService.GetSpeciesObservationChangeAsSpecies(DateTime.Now, false, DateTime.Now, false, maxChangeId, true, Settings.Default.MaxSpeciesObservationsFromMvm); if (webSpeciesObservationChangeTemp.CreatedSpeciesObservations.IsNotEmpty()) { webSpeciesObservationChange.CreatedSpeciesObservations.AddRange(webSpeciesObservationChangeTemp.CreatedSpeciesObservations); } if (webSpeciesObservationChangeTemp.DeletedSpeciesObservationGuids.IsNotEmpty()) { webSpeciesObservationChange.DeletedSpeciesObservationGuids.AddRange(webSpeciesObservationChangeTemp.DeletedSpeciesObservationGuids); } if (webSpeciesObservationChangeTemp.UpdatedSpeciesObservations.IsNotEmpty()) { webSpeciesObservationChange.UpdatedSpeciesObservations.AddRange(webSpeciesObservationChangeTemp.UpdatedSpeciesObservations); } if (maxChangeId < webSpeciesObservationChangeTemp.MaxChangeId) { maxChangeId = webSpeciesObservationChangeTemp.MaxChangeId; } else { // No more species observation changes are available. break; } } // Run all created and updated together as one list. updatedSpeciesObservations = new Dictionary <String, WebSpeciesObservation>(); AddSpeciesObservations(updatedSpeciesObservations, webSpeciesObservationChange.UpdatedSpeciesObservations); AddSpeciesObservations(updatedSpeciesObservations, webSpeciesObservationChange.CreatedSpeciesObservations); List <WebSpeciesObservation> createdAndUpdatedObservations = new List <WebSpeciesObservation>(); createdAndUpdatedObservations.AddRange(updatedSpeciesObservations.Values); // Handle created and updated species observations. int noOfCreated = 0, noOfCreatedErrors = 0; if (createdAndUpdatedObservations.IsNotEmpty()) { foreach (WebSpeciesObservation mvmSpeciesObservation in createdAndUpdatedObservations) { WebData webData = new WebData { DataFields = new List <WebDataField>() }; foreach (WebSpeciesObservationField webSpeciesObservationField in mvmSpeciesObservation.Fields) { WebDataField webDataField = new WebDataField(); webDataField.LoadData(webSpeciesObservationField); webData.DataFields.Add(webDataField); } // Map and add webdata to the 'created collection' speciesObservationChange.CreatedSpeciesObservations.Add(mvmProcess.ProcessObservation(webData, mappings, context)); } connectorServer.UpdateSpeciesObservations(context, speciesObservationChange.CreatedSpeciesObservations, dataProvider, out noOfCreated, out noOfCreatedErrors); } // Handle deleted species observtions. int noOfDeleted = 0, noOfDeletedErrors = 0; speciesObservationChange.DeletedSpeciesObservationGuids = new List <String>(); if (webSpeciesObservationChange.DeletedSpeciesObservationGuids.IsNotEmpty()) { foreach (String mvmSpeciesObservation in webSpeciesObservationChange.DeletedSpeciesObservationGuids) { String id = mvmSpeciesObservation.Substring(mvmSpeciesObservation.LastIndexOf(':') + 1); speciesObservationChange.DeletedSpeciesObservationGuids.Add(id); } connectorServer.DeleteSpeciesObservations(context, speciesObservationChange.DeletedSpeciesObservationGuids, dataProvider, out noOfDeleted, out noOfDeletedErrors); } Debug.WriteLine(maxChangeId + " : count created: " + noOfCreated); context.GetSpeciesObservationDatabase().LogHarvestRead( context, dataProvider, changedFrom, changedTo, stopwatch.ElapsedMilliseconds, noOfCreated, noOfCreatedErrors, 0, 0, noOfDeleted, noOfDeletedErrors, maxChangeId); stopwatch.Stop(); if (maxChangeId > 0) { context.GetSpeciesObservationDatabase().SetMaxChangeId(dataProvider.Id, maxChangeId); } // else: No changes was retrieved. There are no more changes available right now. return(false); }