/// <summary> /// Gets all data series that match the /// specified criteria and are within the geographic polygon /// </summary> /// <param name="polygons">one or multiple polygons</param> /// <param name="keywords">array of keywords. If set to null, /// results will not be filtered by keyword.</param> /// <param name="startDate">start date. If set to null, results will not be filtered by start date.</param> /// <param name="endDate">end date. If set to null, results will not be filtered by end date.</param> /// <param name="serviceIDs">array of serviceIDs provided by GetServicesInBox. /// <param name="bgWorker">The background worker (may be null) for reporting progress</param> /// <param name="e">The results of the search (convert to DataTable)</param> /// If set to null, results will not be filtered by web service.</param> /// <returns>A list of data series matching the specified criteria</returns> public void GetSeriesCatalogInPolygon(IList <IFeature> polygons, string[] keywords, DateTime startDate, DateTime endDate, int[] serviceIDs, IProgressHandler bgWorker, DoWorkEventArgs e) { double tileWidth = 1.0; //the initial tile width is set to 1 degree double tileHeight = 1.0; //the initial tile height is set to 1 degree //(1): Get the union of the polygons if (polygons.Count == 0) { throw new ArgumentException("The number of polygons must be greater than zero."); } // Check for cancel bgWorker.CheckForCancel(); if (polygons.Count > 1) { bgWorker.ReportProgress(0, "Processing Polygons"); } //get the list of series var fullSeriesList = new List <SeriesDataCart>(); foreach (IFeature polygon in polygons) { //Split the polygon area bounding box into 1x1 decimal degree tiles IEnvelope env = polygon.Envelope; Box extentBox = new Box(env.Left(), env.Right(), env.Bottom(), env.Top()); IList <Box> tiles = SearchHelper.CreateTiles(extentBox, tileWidth, tileHeight); int numTiles = tiles.Count; for (int i = 0; i < numTiles; i++) { Box tile = tiles[i]; bgWorker.CheckForCancel(); // Do the web service call IEnumerable <SeriesDataCart> tileSeriesList = GetSeriesCatalogForBox(tile.XMin, tile.XMax, tile.YMin, tile.YMax, keywords, startDate, endDate, serviceIDs); // Clip the points by polygon IEnumerable <SeriesDataCart> seriesInPolygon = SearchHelper.ClipByPolygon(tileSeriesList, polygon); fullSeriesList.AddRange(seriesInPolygon); // Report progress { string message = fullSeriesList.Count.ToString(); int percentProgress = (i * 100) / numTiles + 1; bgWorker.ReportProgress(percentProgress, message); } } } //(4) Create the Feature Set SearchResult resultFs = null; if (fullSeriesList.Count > 0) { bgWorker.ReportProgress(0, "Calculating Points"); resultFs = SearchHelper.ToFeatureSetsByDataSource(fullSeriesList); } // (5) Final Background worker updates if (e != null) { bgWorker.CheckForCancel(); // Report progress bgWorker.ReportProgress(100, "Search Finished"); e.Result = resultFs; } }
/// <summary> /// Gets all search result that match the /// specified criteria and are within the specific rectangle /// </summary> /// <param name="keywords">array of keywords. If set to null, /// results will not be filtered by keyword.</param> /// <param name="startDate">start date. If set to null, results will not be filtered by start date.</param> /// <param name="endDate">end date. If set to null, results will not be filtered by end date.</param> /// <param name="e">The results of the search (convert to DataTable)</param> /// <returns>A list of data series matching the specified criteria</returns> public void GetSeriesCatalogInRectangle(double xMin, double XMax, double YMin, double YMax, string[] keywords, DateTime startDate, DateTime endDate, int[] serviceIDs, IProgressHandler bgWorker, DoWorkEventArgs e) { if (bgWorker == null) { throw new ArgumentNullException("bgWorker"); } double tileWidth = 1.0; //the initial tile width is set to 1 degree double tileHeight = 1.0; //the initial tile height is set to 1 degree bgWorker.CheckForCancel(); //get the list of series var fullSeriesList = new List <SeriesDataCart>(); //Split the polygon area bounding box into 1x1 decimal degree tiles Box extentBox = new Box(xMin, XMax, YMin, YMax); IList <Box> tiles = SearchHelper.CreateTiles(extentBox, tileWidth, tileHeight); int numTiles = tiles.Count; for (int i = 0; i < numTiles; i++) { Box tile = tiles[i]; bgWorker.CheckForCancel(); // Do the web service call //IList<SeriesDataCart> tileSeriesList = new List<SeriesMetadata>(); IEnumerable <SeriesDataCart> tileSeriesList = GetSeriesCatalogForBox(tile.XMin, tile.XMax, tile.YMin, tile.YMax, keywords, startDate, endDate, serviceIDs); fullSeriesList.AddRange(tileSeriesList); // Report progress { string message = fullSeriesList.Count.ToString(); int percentProgress = (i * 100) / numTiles + 1; bgWorker.ReportProgress(percentProgress, message); } } //(4) Create the Feature Set SearchResult resultFs = null; if (fullSeriesList.Count > 0) { bgWorker.ReportProgress(0, "Calculating Points"); resultFs = SearchHelper.ToFeatureSetsByDataSource(fullSeriesList); } // (5) Final Background worker updates if (e != null) { bgWorker.CheckForCancel(); // Report progress bgWorker.ReportProgress(100, "Search Finished"); e.Result = resultFs; } }
public async Task <bool> BatchRunAsync(CancellationToken cancellationToken = default) { m_JournalWriter.WriteLine($"Batch macro running started"); var batchStartTime = DateTime.Now; TimeSpan?timeout = null; if (m_Job.Timeout > 0) { timeout = TimeSpan.FromSeconds(m_Job.Timeout); } var worker = m_WorkerFact.Invoke(timeout); worker.Retry += OnRetry; worker.Timeout += OnTimeout; m_CurrentContext = new BatchJobContext() { Job = m_Job }; var jobResult = false; try { await TaskEx.Run(() => { m_JournalWriter.WriteLine($"Collecting files for processing"); var app = EnsureApplication(m_CurrentContext, cancellationToken); var allFiles = PrepareJobScope(app, m_Job.Input, m_Job.Filters, m_Job.Macros); m_JournalWriter.WriteLine($"Running batch processing for {allFiles.Length} file(s)"); m_ProgressHandler.SetJobScope(allFiles, batchStartTime); if (!allFiles.Any()) { throw new UserException("Empty job. No files matching specified filter"); } var curBatchSize = 0; for (int i = 0; i < allFiles.Length; i++) { var curAppPrc = m_CurrentContext.CurrentApplicationProcess?.Id; m_CurrentContext.CurrentJobItem = allFiles[i]; var res = TryProcessFile(m_CurrentContext, worker, cancellationToken); TryCloseDocument(m_CurrentContext.CurrentDocument); m_CurrentContext.CurrentDocument = null; m_ProgressHandler?.ReportProgress(m_CurrentContext.CurrentJobItem, res); if (!res && !m_Job.ContinueOnError) { throw new UserException("Cancelling the job. Set 'Continue On Error' option to continue job if file failed"); } if (m_CurrentContext.CurrentApplicationProcess?.Id != curAppPrc) { curBatchSize = 1; } else { curBatchSize++; } if (m_Job.BatchSize > 0 && curBatchSize >= m_Job.BatchSize) { m_JournalWriter.WriteLine("Closing application as batch size reached the limit"); TryShutDownApplication(m_CurrentContext.CurrentApplicationProcess); curBatchSize = 0; } } }, new StaTaskScheduler(m_Logger)).ConfigureAwait(false); jobResult = true; } catch (OperationCanceledException) { throw new JobCancelledException(); } finally { TryShutDownApplication(m_CurrentContext.CurrentApplicationProcess); } var duration = DateTime.Now.Subtract(batchStartTime); m_ProgressHandler.ReportCompleted(duration); m_JournalWriter.WriteLine($"Batch running completed in {duration.ToString(@"hh\:mm\:ss")}"); return(jobResult); }
public void DeriveInsertDataValues(double A, double B, double C, double D, double E, double F, DataTable dt, long newSeriesID, long sourceSeriesID, bool isAlgebraic, IProgressHandler progressHandler) { const int chunkLength = 400; var nodatavalue = GetNoDataValueForSeriesVariable(newSeriesID); const string insertQuery = "INSERT INTO DataValues(ValueID, SeriesID, DataValue, ValueAccuracy, LocalDateTime, UtcOffset, DateTimeUtc, OffsetValue, OffsetTypeID, CensorCode, QualifierID, SampleID, FileID) " + "VALUES ({0}, {1}, {2}, {3}, '{4}', {5}, '{6}', {7}, {8}, '{9}', {10}, {11}, {12});"; var index = 0; while (index != dt.Rows.Count - 1) { //Save values by chunks var newValueID = DbOperations.GetNextID("DataValues", "ValueID"); var query = new StringBuilder("BEGIN TRANSACTION; "); for (int i = 0; i < chunkLength; i++) { // Calculating value double newvalue = 0.0; if (isAlgebraic) { var currentvalue = Convert.ToDouble(dt.Rows[index]["DataValue"]); if (currentvalue != nodatavalue) { //NOTE:Equation = Fx ^ 5 + Ex ^ 4 + Dx ^ 3 + Cx ^ 2 + Bx + A newvalue = (F * (Math.Pow(currentvalue, 5))) + (E * (Math.Pow(currentvalue, 4))) + (D * (Math.Pow(currentvalue, 3))) + (C * (Math.Pow(currentvalue, 2))) + (B * currentvalue) + A; newvalue = Math.Round(newvalue, 5); } else { newvalue = nodatavalue; } } else { newvalue = Convert.ToDouble(dt.Rows[index]["DataValue"]); } var row = dt.Rows[index]; query.AppendFormat(insertQuery, newValueID + i, newSeriesID, newvalue, row["ValueAccuracy"].ToString() == "" ? "NULL" : row["ValueAccuracy"].ToString(), Convert.ToDateTime(row["LocalDateTime"]).ToString("yyyy-MM-dd HH:mm:ss"), row["UTCOffset"].ToString(), Convert.ToDateTime(row["DateTimeUTC"]).ToString("yyyy-MM-dd HH:mm:ss"), row["OffsetValue"].ToString() == "" ? "NULL" : row["OffsetValue"].ToString(), row["OffsetTypeID"].ToString() == "" ? "NULL" : row["OffsetTypeID"].ToString(), row["CensorCode"].ToString(), row["QualifierID"].ToString() == "" ? "NULL" : row["QualifierID"].ToString(), row["SampleID"].ToString() == "" ? "NULL" : row["SampleID"].ToString(), row["FileID"].ToString() == "" ? "NULL" : row["FileID"].ToString()); query.AppendLine(); if (index == dt.Rows.Count - 1) { break; } index = index + 1; } query.AppendLine("COMMIT;"); DbOperations.ExecuteNonQuery(query.ToString()); progressHandler.ReportProgress(index, null); } }
public void DeriveInsertAggregateDataValues(DataTable dt, long newSeriesID, DateTime currentdate, DateTime lastdate, DeriveAggregateMode mode, DeriveComputeMode computeMode, double nodatavalue, IProgressHandler progressHandler) { const string insertQuery = "INSERT INTO DataValues(ValueID, SeriesID, DataValue, ValueAccuracy, LocalDateTime, UtcOffset, DateTimeUtc, OffsetValue, OffsetTypeID, CensorCode, QualifierID, SampleID, FileID) " + "VALUES ({0}, {1}, {2}, {3}, '{4}', {5}, '{6}', {7}, {8}, '{9}', {10}, {11}, {12});"; const int chunkLength = 400; var index = 0; while (currentdate <= lastdate) { // Save values by chunks var newValueID = DbOperations.GetNextID("DataValues", "ValueID"); var query = new StringBuilder("BEGIN TRANSACTION; "); for (int i = 0; i <= chunkLength - 1; i++) { double newvalue = 0.0; string sqlString = string.Empty; double UTC = 0.0; switch (mode) { case DeriveAggregateMode.Daily: sqlString = "LocalDateTime >= '" + currentdate.ToString(CultureInfo.InvariantCulture) + "' AND LocalDateTime <= '" + currentdate.AddDays(1).AddMilliseconds(-1).ToString(CultureInfo.InvariantCulture) + "' AND DataValue <> " + nodatavalue.ToString(CultureInfo.InvariantCulture); break; case DeriveAggregateMode.Monthly: sqlString = "LocalDateTime >= '" + currentdate.ToString(CultureInfo.InvariantCulture) + "' AND LocalDateTime <= '" + currentdate.AddMonths(1).AddMilliseconds(-1).ToString(CultureInfo.InvariantCulture) + "' AND DataValue <> " + nodatavalue.ToString(CultureInfo.InvariantCulture); break; case DeriveAggregateMode.Quarterly: sqlString = "LocalDateTime >= '" + currentdate.ToString(CultureInfo.InvariantCulture) + "' AND LocalDateTime <= '" + currentdate.AddMonths(3).AddMilliseconds(-1).ToString( CultureInfo.InvariantCulture) + "' AND DataValue <> " + nodatavalue.ToString(CultureInfo.InvariantCulture); break; } try { switch (computeMode) { case DeriveComputeMode.Maximum: newvalue = Convert.ToDouble(dt.Compute("Max(DataValue)", sqlString)); break; case DeriveComputeMode.Minimum: newvalue = Convert.ToDouble(dt.Compute("MIN(DataValue)", sqlString)); break; case DeriveComputeMode.Average: newvalue = Convert.ToDouble(dt.Compute("AVG(DataValue)", sqlString)); break; case DeriveComputeMode.Sum: newvalue = Convert.ToDouble(dt.Compute("Sum(DataValue)", sqlString)); break; } UTC = Convert.ToDouble(dt.Compute("AVG(UTCOffset)", sqlString)); } catch (Exception) { newvalue = nodatavalue; } query.AppendFormat(insertQuery, newValueID + i, newSeriesID, newvalue, 0, Convert.ToDateTime(dt.Rows[index]["LocalDateTime"]).ToString("yyyy-MM-dd HH:mm:ss"), UTC.ToString(CultureInfo.InvariantCulture), currentdate.AddHours(UTC).ToString("yyyy-MM-dd HH:mm:ss"), "NULL", "NULL", "nc", "NULL", "NULL", "NULL"); query.AppendLine(); switch (mode) { case DeriveAggregateMode.Daily: currentdate = currentdate.AddDays(1); break; case DeriveAggregateMode.Monthly: currentdate = currentdate.AddMonths(1); break; case DeriveAggregateMode.Quarterly: currentdate = currentdate.AddMonths(3); break; } if (currentdate > lastdate) { break; } index = index + 1; //Report progress progressHandler.ReportProgress(index - 1, null); } query.AppendLine("COMMIT;"); DbOperations.ExecuteNonQuery(query.ToString()); progressHandler.ReportProgress(index - 1, null); } }
public async Task <bool> BatchRun(BatchJob opts, CancellationToken cancellationToken = default) { m_UserLogger.WriteLine($"Batch macro running started"); var batchStartTime = DateTime.Now; var allFiles = PrepareJobScope(opts.Input, opts.Filters, opts.Macros).ToArray(); m_UserLogger.WriteLine($"Running batch processing for {allFiles.Length} file(s)"); m_ProgressHandler.SetJobScope(allFiles, batchStartTime); if (!allFiles.Any()) { throw new UserMessageException("Empty job. No files matching specified filter"); } TimeSpan timeout = default; if (opts.Timeout > 0) { timeout = TimeSpan.FromSeconds(opts.Timeout); } IXApplication app = null; Process appPrc = null; if (cancellationToken != default) { cancellationToken.Register(() => { m_UserLogger.WriteLine($"Cancelled by the user"); TryShutDownApplication(appPrc); }); } var jobResult = false; try { await Task.Run(() => { var curBatchSize = 0; for (int i = 0; i < allFiles.Length; i++) { var curAppPrc = appPrc?.Id; var curFile = allFiles[i]; var res = AttemptProcessFile(ref app, ref appPrc, curFile, opts, cancellationToken); m_ProgressHandler?.ReportProgress(curFile, res); if (!res && !opts.ContinueOnError) { throw new UserMessageException("Cancelling the job. Set 'Continue On Error' option to continue job if file failed"); } if (appPrc?.Id != curAppPrc) { curBatchSize = 1; } else { curBatchSize++; } if (opts.BatchSize > 0 && curBatchSize >= opts.BatchSize && !cancellationToken.IsCancellationRequested) { m_UserLogger.WriteLine("Closing application as batch size reached the limit"); TryShutDownApplication(appPrc); curBatchSize = 0; } } }).ConfigureAwait(false); jobResult = true; } finally { TryShutDownApplication(appPrc); } var duration = DateTime.Now.Subtract(batchStartTime); m_ProgressHandler.ReportCompleted(duration); m_UserLogger.WriteLine($"Batch running completed in {duration.ToString(@"hh\:mm\:ss")}"); return(jobResult); }