/// <summary>
/// Extracts a Stream ready for schema parsing from a web response stream. With this and GetValuesAsync, the request executes asynchronously, while the parsing blocks (desired behavior).
/// http://stackoverflow.com/questions/10565090/getting-the-response-of-a-asynchronous-httpwebrequest
/// </summary>
/// <param name="response"></param>
/// <returns></returns>
private static Stream ReadStreamFromResponse(WebResponse response)
{
Stream stream = response.GetResponseStream();
//Copy stream contents to a new memory stream and return the memory stream...
MemoryStream ms = new MemoryStream();
stream.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
#if (DEBUG)
//Here the stream contains data...
if (EnvironmentContext.LocalEnvironment())
{
using (System.IO.FileStream output = new System.IO.FileStream(@"C:\CUAHSI\ReadFromStreamResponse.xml", FileMode.Create))
{
ms.CopyTo(output);
output.Flush();
ms.Seek(0, SeekOrigin.Begin);
}
}
#endif
return(ms);
}
private void writeMemoryStreamToFile(string filePathAndName, MemoryStream ms)
{
//Validate/initialize input parameters...
if (String.IsNullOrWhiteSpace(filePathAndName) || null == ms)
{
return; //Input parameter(s) invalid - return early
}
try
{
if (EnvironmentContext.LocalEnvironment())
{
//Position memory stream...
ms.Seek(0, SeekOrigin.Begin);
//Create file...
using (System.IO.FileStream output = new System.IO.FileStream(filePathAndName, FileMode.OpenOrCreate))
{
//Create XmlReader on memory stream...
using (var reader = XmlReader.Create(ms))
{
//Create XmlWriter on file...
using (XmlWriter writer = XmlWriter.Create(output))
{
//Write contents of reader to file and flush...
writer.WriteNode(reader, true);
output.Flush();
//Re-position memory stream...
ms.Seek(0, SeekOrigin.Begin);
}
}
}
}
}
catch (Exception ex)
{
//Take no action...
string msg = ex.Message;
}
}
private List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart> GetSeriesListForExtent(Extent extent, IEnumerable <string> keywords, double tileWidth, double tileHeight,
DateTime startDate, DateTime endDate, ICollection <WebServiceNode> serviceIDs,
//BusinessObjects.Models.IProgressHandler bgWorker,
Func <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart, bool> seriesFilter)
{
var servicesToSearch = new List <Tuple <WebServiceNode[], Extent> >();
if (serviceIDs.Count > 0)
{
foreach (var webService in serviceIDs)
{
if (webService.ServiceBoundingBox == null)
{
servicesToSearch.Add(new Tuple <WebServiceNode[], Extent>(new[] { webService }, extent));
continue;
}
const double eps = 0.05; //tolerance (0.05 deg) used for services whose bounding box is one point
var wsBox = webService.ServiceBoundingBox;
var wsExtent = new Extent(wsBox.XMin - eps, wsBox.YMin - eps, wsBox.XMax + eps, wsBox.YMax + eps);
if (wsExtent.Intersects(extent))
{
servicesToSearch.Add(new Tuple <WebServiceNode[], Extent>(new[] { webService }, wsExtent.Intersection(extent)));
}
}
}
else
{
servicesToSearch.Add(new Tuple <WebServiceNode[], Extent>(new WebServiceNode[] { }, extent));
}
var servicesWithExtents = new List <Tuple <WebServiceNode[], List <Extent> > >(servicesToSearch.Count);
int totalTilesCount = 0;
foreach (var wsInfo in servicesToSearch)
{
var tiles = SearchHelper.CreateTiles(wsInfo.Item2, tileWidth, tileHeight);
servicesWithExtents.Add(new Tuple <WebServiceNode[], List <Extent> >(wsInfo.Item1, tiles));
totalTilesCount += tiles.Count;
}
var fullSeriesList = new List <List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart> >();
long currentTileIndex = 0;
int tilesFinished = 0;
totalSeriesCount = 0;
//bgWorker.ReportProgress(0, "0 Series found");
CancellationTokenSource cts = new CancellationTokenSource();
var serviceLoopOptions = new ParallelOptions
{
//CancellationToken = bgWorker.CancellationToken,
#if (DEBUG)
MaxDegreeOfParallelism = EnvironmentContext.LocalEnvironment() ? 1 : 2
#else
MaxDegreeOfParallelism = 2
#endif
};
var tileLoopOptions = new ParallelOptions
{
//CancellationToken = bgWorker.CancellationToken,
CancellationToken = cts.Token,
// Note: currently HIS Central returns timeout if many requests are sent in the same time.
// To test set MaxDegreeOfParallelism = -1
#if (DEBUG)
MaxDegreeOfParallelism = EnvironmentContext.LocalEnvironment() ? 1 : 4
#else
MaxDegreeOfParallelism = 4
#endif
};
try
{
Parallel.ForEach(servicesWithExtents, serviceLoopOptions, wsInfo =>
{
//bgWorker.CheckForCancel();
var ids = wsInfo.Item1.Select(item => item.ServiceID).ToArray();
var tiles = wsInfo.Item2;
try
{
Parallel.ForEach(tiles, tileLoopOptions, tile =>
{
var current = Interlocked.Add(ref currentTileIndex, 1);
//bgWorker.CheckForCancel();
// Do the web service call
var tileSeriesList = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
if (SearchSettings.AndSearch == true)
{
//CHANGES FOR "AND" SEARCH
var totalTileSeriesList = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
var tileSeriesList2 = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
var tileSeriesList3 = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
var tileSeriesList4 = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
SeriesComparer sc = new SeriesComparer();
for (int i = 0; i < keywords.Count(); i++)
{
String keyword = keywords.ElementAt(i);
string sampleMedium = String.Empty;
string dataType = String.Empty;
string valueType = string.Empty;
var series = GetSeriesCatalogForBox(tile.MinX, tile.MaxX, tile.MinY, tile.MaxY, sampleMedium, dataType, valueType, keyword, startDate, endDate, ids, current, totalTilesCount);
totalTileSeriesList.AddRange(series);
if (tileSeriesList.Count() == 0)
{
if (i == 0)
{
tileSeriesList.AddRange(series);
}
else
{
break;
}
}
else
{
tileSeriesList2.AddRange(tileSeriesList.Intersect(series, sc));
tileSeriesList.Clear();
tileSeriesList.AddRange(tileSeriesList2);
tileSeriesList2.Clear();
}
}
for (int i = 0; i < tileSeriesList.Count(); i++)
{
tileSeriesList4 = totalTileSeriesList.Where(item => (item.SiteName.Equals(tileSeriesList.ElementAt(i).SiteName))).ToList();
tileSeriesList3.AddRange(tileSeriesList4);
}
tileSeriesList = tileSeriesList3;
}
else
{
tileSeriesList = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>();
foreach (var keyword in keywords)
{
string sampleMedium = String.Empty;
string dataType = String.Empty;
string valueType = string.Empty;
var series = GetSeriesCatalogForBox(tile.MinX, tile.MaxX, tile.MinY, tile.MaxY, sampleMedium, dataType, valueType, keyword, startDate, endDate, ids, current, totalTilesCount);
tileSeriesList.AddRange(series);
}
}
//END CHANGES FOR "AND" SEARCH
//bgWorker.CheckForCancel();
if (tileSeriesList.Count > 0)
{
var filtered = tileSeriesList.Where(seriesFilter).ToList();
if (filtered.Count > 0)
{
lock (_lockGetSeries)
{
totalSeriesCount += filtered.Count;
fullSeriesList.Add(filtered);
}
}
}
// Report progress
var currentFinished = Interlocked.Add(ref tilesFinished, 1);
if (totalSeriesCount > maxAllowedTimeseriesReturn)
{
//Maximum time series exceeded - register a delegate to add an exception to the aggregate exception returned by the task cancellation processing...
cts.Token.Register(() => {
string errorMessage = String.Format("Search returned more than {0:#,###0} timeseries and was canceled. Please limit search area and/or Keywords.", maxAllowedTimeseriesReturn);
InvalidOperationException exp = new InvalidOperationException(errorMessage);
throw exp;
});
cts.Cancel();
}
var message = string.Format("{0} Series found", totalSeriesCount);
var percentProgress = (currentFinished * 100) / totalTilesCount;
//bgWorker.ReportProgress(percentProgress, message);
});
}
catch (OperationCanceledException oex)
{
throw oex;
}
});
// Collect all series into result list
var result = new List <BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>(totalSeriesCount);
fullSeriesList.ForEach(result.AddRange);
return(result);
}
catch (OperationCanceledException oex)
{
throw oex;
}
}
public ActionResult updateMarkers(FormCollection collection)
{
if (bFirstCall)
{
//First time called since last web site restart - log selected <appSettings> values...
bFirstCall = false;
dblogcontext.clearParameters();
dblogcontext.clearReturns();
dblogcontext.addParameter("ServiceUrl", ConfigurationManager.AppSettings["ServiceUrl"]);
dblogcontext.addParameter("ServiceUrl_1_1_EndPoint", ConfigurationManager.AppSettings["ServiceUrl1_1_EndPoint"]);
dblogcontext.addParameter("ByuUrl", ConfigurationManager.AppSettings["ByuUrl"]);
dblogcontext.addParameter("MaxClustercount", ConfigurationManager.AppSettings["MaxClustercount"].ToString());
dblogcontext.addParameter("maxAllowedTimeseriesReturn", ConfigurationManager.AppSettings["maxAllowedTimeseriesReturn"].ToString());
dblogcontext.addParameter("maxCombinedExportValues", ConfigurationManager.AppSettings["maxCombinedExportValues"].ToString());
dblogcontext.addParameter("blobContainer", ConfigurationManager.AppSettings["blobContainer"]);
dblogcontext.addParameter("aspnet:MaxJsonDeserializerMembers", ConfigurationManager.AppSettings["aspnet:MaxJsonDeserializerMembers"].ToString());
dblogcontext.addParameter("currentVersion", ConfigurationManager.AppSettings["currentVersion"].ToString());
DateTime dtNow = DateTime.UtcNow;
dblogcontext.createLogEntry(System.Web.HttpContext.Current, dtNow, dtNow, "updateMarkers(...)", "first call - selected appSettings values...", Level.Info);
}
var searchSettings = new SearchSettings();
string markerjSON = string.Empty;
//get map geometry
double xMin, xMax, yMin, yMax;
int zoomLevel;
int CLUSTERWIDTH = 50; //Cluster region width, all pin within this area are clustered
int CLUSTERHEIGHT = 50; //Cluster region height, all pin within this area are clustered
int CLUSTERINCREMENT = 5; //increment for clusterwidth
int MINCLUSTERDISTANCE = 25;
int MAXCLUSTERCOUNT = Convert.ToInt32(ConfigurationManager.AppSettings["MaxClustercount"].ToString()); //maximum ammount of clustered markers
UniversalTypeConverter.TryConvertTo <double>(collection["xMin"], out xMin);
UniversalTypeConverter.TryConvertTo <double>(collection["xMax"], out xMax);
UniversalTypeConverter.TryConvertTo <double>(collection["yMin"], out yMin);
UniversalTypeConverter.TryConvertTo <double>(collection["yMax"], out yMax);
Box box = new Box(xMin, xMax, yMin, yMax);
UniversalTypeConverter.TryConvertTo <int>(collection["zoomLevel"], out zoomLevel);
var activeWebservices = new List <WebServiceNode>();
//Attempt to retrieve filtered timeseries ids, if indicated
string strFilterAndSearchCriteria = collection["filterAndSearchCriteria"];
clientFilterAndSearchCriteria filterAndSearchCriteria = null;
if (!String.IsNullOrWhiteSpace(strFilterAndSearchCriteria))
{
filterAndSearchCriteria = JsonConvert.DeserializeObject <clientFilterAndSearchCriteria>(strFilterAndSearchCriteria);
}
//if it is a new request
if (collection["isNewRequest"].ToString() == "true")
{
bool canConvert = false;
var keywords = Regex.Split(collection["keywords"], @"##");
var tileWidth = 10;
var tileHeight = 10;
List <int> webServiceIds = null;
try
{
//Increase date range by one day to accomodate one day searches, if indicated...
//Set begin date time to 00:00:00, set end date time to 23:59:59
searchSettings.DateSettings.StartDate = Convert.ToDateTime(collection["startDate"]);
searchSettings.DateSettings.StartDate = searchSettings.DateSettings.StartDate.Date.AddHours(0).AddMinutes(0).AddSeconds(0);
searchSettings.DateSettings.EndDate = Convert.ToDateTime(collection["endDate"]);
if (searchSettings.DateSettings.StartDate.Date == searchSettings.DateSettings.EndDate.Date)
{
searchSettings.DateSettings.EndDate = searchSettings.DateSettings.EndDate.Date.AddDays(1);
}
searchSettings.DateSettings.EndDate = searchSettings.DateSettings.EndDate.Date.AddHours(23).AddMinutes(59).AddSeconds(59);
//Convert to int Array
if (collection["services"].Length > 0)
{
webServiceIds = collection["services"].Split(',').Select(s => Convert.ToInt32(s)).ToList();
}
var dataWorker = new DataWorker();
var allWebservices = dataWorker.getWebServiceList();
//filter list
if (webServiceIds != null)
{
activeWebservices = dataWorker.filterWebservices(allWebservices, webServiceIds);
}
Session["webServiceList"] = allWebservices;
//Clear parameters...
dblogcontext.clearParameters();
dberrorcontext.clearParameters();
//Add call parameters...
dblogcontext.addParameter("box", box);
dberrorcontext.addParameter("box", box);
if (1 == keywords.Length && "" == keywords[0])
{
dblogcontext.addParameter("keywords", "All");
dberrorcontext.addParameter("keywords", "All");
}
else
{
StringBuilder sb1 = new StringBuilder();
foreach (string keyword in keywords)
{
sb1.AppendFormat("{0}, ", keyword);
}
dblogcontext.addParameter("keywords", sb1.ToString());
dberrorcontext.addParameter("keywords", sb1.ToString());
}
dblogcontext.addParameter("tileWidth", tileWidth);
dberrorcontext.addParameter("tileWidth", tileWidth);
dblogcontext.addParameter("tileHeight", tileHeight);
dberrorcontext.addParameter("tileHeight", tileHeight);
dblogcontext.addParameter("startDate", searchSettings.DateSettings.StartDate);
dberrorcontext.addParameter("startDate", searchSettings.DateSettings.StartDate);
dblogcontext.addParameter("endDate", searchSettings.DateSettings.EndDate);
dberrorcontext.addParameter("endDate", searchSettings.DateSettings.EndDate);
if (0 >= activeWebservices.Count)
{
dblogcontext.addParameter("activeWebServices", "All");
dberrorcontext.addParameter("activeWebServices", "All");
}
else
{
StringBuilder sb1 = new StringBuilder();
activeWebservices.ForEach(wsn => sb1.AppendFormat("{0},", wsn.Title));
dblogcontext.addParameter("activeWebServices", sb1.ToString());
dberrorcontext.addParameter("activeWebServices", sb1.ToString());
}
DateTime startDtUtc = DateTime.UtcNow;
var series = dataWorker.getSeriesData(box, keywords.ToArray(), tileWidth, tileHeight,
searchSettings.DateSettings.StartDate,
searchSettings.DateSettings.EndDate,
activeWebservices);
DateTime endDtUtc = DateTime.UtcNow;
//Clear returns
dblogcontext.clearReturns();
//Add returned series count...
dblogcontext.addReturn("seriesCount", series.Count);
//Create log entry...
dblogcontext.createLogEntry(System.Web.HttpContext.Current, startDtUtc, endDtUtc, "updateMarkers(...)", "calls dataWorker.getSeriesData(...)", Level.Info);
var list = new List <TimeSeriesViewModel>();
if (!EnvironmentContext.LocalEnvironment())
{
//Non-local environment - do not attempt to create debug files...
//Watch out for similar loop code in 'else' block...
if (series.Count > 0)
{
for (int i = 0; i < series.Count; i++)
{
var tvm = new TimeSeriesViewModel();
tvm = mapDataCartToTimeseries(series[i], i);
list.Add(tvm);
}
}
}
else
{
//Local environment - create debug files, if indicated...
#if (DEBUG)
//BCC - Test - write data cart and time series objects to files...
using (System.IO.StreamWriter swSdc = System.IO.File.CreateText(@"C:\CUAHSI\SeriesDataCart.json"))
{
using (System.IO.StreamWriter swTsvm = System.IO.File.CreateText(@"C:\CUAHSI\TimeSeriesViewModel.json"))
{
JsonSerializer jsonser = new JsonSerializer();
swSdc.Write('['); //Write start of array...
swTsvm.Write('[');
#endif
//Watch out for similar loop code in 'if' block...
if (series.Count > 0)
{
for (int i = 0; i < series.Count; i++)
{
var tvm = new TimeSeriesViewModel();
tvm = mapDataCartToTimeseries(series[i], i);
list.Add(tvm);
#if (DEBUG)
jsonser.Serialize(swSdc, series[i]);
jsonser.Serialize(swTsvm, tvm);
if ((i + 1) < series.Count)
{
swSdc.Write(','); //Separate array element...
swTsvm.Write(',');
}
#endif
}
}
#if (DEBUG)
swSdc.Write(']'); //Write end of array...
swTsvm.Write(']');
}
}
#endif
}
var markerClustererHelper = new MarkerClustererHelper();
//save list for later
Session["Series"] = list;
//transform list int clusteredpins
var pins = transformSeriesDataCartIntoClusteredPin(list, filterAndSearchCriteria);
var clusteredPins = markerClustererHelper.clusterPins(pins, CLUSTERWIDTH, CLUSTERHEIGHT, CLUSTERINCREMENT, zoomLevel, MAXCLUSTERCOUNT, MINCLUSTERDISTANCE);
Session["ClusteredPins"] = clusteredPins;
var centerPoint = new LatLong(0, 0);
markerjSON = markerClustererHelper.createMarkersGEOJSON(clusteredPins, zoomLevel, centerPoint, "");
}
catch (Exception ex)
{
//NOTE: Override 'standard' IIS error handling since we are using 'standard' HTTP error codes - RequestEntityTooLarge and RequestTimeout...
//Sources: http://stackoverflow.com/questions/22071211/when-performing-post-via-ajax-bad-request-is-returned-instead-of-the-json-resul
// http://stackoverflow.com/questions/3993941/in-iis7-5-what-module-removes-the-body-of-a-400-bad-request/4029197#4029197
// http://weblog.west-wind.com/posts/2009/Apr/29/IIS-7-Error-Pages-taking-over-500-Errors
//Find the 'inner-most' exception...
while (null != ex.InnerException)
{
ex = ex.InnerException;
}
if (typeof(WebException) == ex.GetType())
{
//Web exception - return the error message...
WebException wex = (WebException)ex;
Response.StatusCode = (int)wex.Status;
Response.StatusDescription = wex.Message;
Response.TrySkipIisCustomErrors = true; //Tell IIS to use your error text not the 'standard' error text!!
//ALSO clues jQuery to add the parsed responseJSON object to the jqXHR object!!
dberrorcontext.createLogEntry(System.Web.HttpContext.Current, DateTime.UtcNow, "updateMarkers(...)", wex, "Web Exception: " + wex.Message);
return(Json(new { Message = wex.Message }, "application/json"));
}
else if (typeof(System.InvalidOperationException) == ex.GetType())
{
//Recover the returned error message...
Response.StatusCode = (int)HttpStatusCode.RequestEntityTooLarge;
Response.StatusDescription = ex.Message;
Response.TrySkipIisCustomErrors = true; //Tell IIS to use your error text not the 'standard' error text!!
//ALSO clues jQuery to add the parsed responseJSON object to the jqXHR object!!
dberrorcontext.createLogEntry(System.Web.HttpContext.Current, DateTime.UtcNow, "updateMarkers(...)", ex, "NON-Web Exception" + ex.Message);
return(Json(new { Message = ex.Message }, "application/json"));
}
else
{
//Assume a timeout has occurred...
string message = "The execution of the search took too long. Please limit search area and/or Keywords.";
Response.StatusCode = (int)HttpStatusCode.RequestTimeout;
Response.StatusDescription = message;
Response.TrySkipIisCustomErrors = true; //Tell IIS to use your error text not the 'standard' error text!!
//ALSO clues jQuery to add the parsed responseJSON object to the jqXHR object!!
dberrorcontext.createLogEntry(System.Web.HttpContext.Current, DateTime.UtcNow, "updateMarkers(...)", ex, "Defaults to timeout exception: " + message);
return(Json(new { Message = message }, "application/json"));
}
}
//var session2 =(List<BusinessObjects.Models.SeriesDataCartModel.SeriesDataCart>) Session["Series"];
}
else
{
//BCC - 19-Nov-2015 - GitHub Issues #67 - Application unresponsive after session timeout and zoom out...
var retrievedSeries = (List <TimeSeriesViewModel>)Session["Series"];
if (null != retrievedSeries) //If a session timeout has occurred, the new session object will not contain the 'Series' element!!
{
var markerClustererHelper = new MarkerClustererHelper();
//transform list int clusteredpins
var pins = transformSeriesDataCartIntoClusteredPin(retrievedSeries, filterAndSearchCriteria);
var clusteredPins = markerClustererHelper.clusterPins(pins, CLUSTERWIDTH, CLUSTERHEIGHT, CLUSTERINCREMENT, zoomLevel, MAXCLUSTERCOUNT, MINCLUSTERDISTANCE);
Session["ClusteredPins"] = clusteredPins;
var centerPoint = new LatLong(0, 0);
markerjSON = markerClustererHelper.createMarkersGEOJSON(clusteredPins, zoomLevel, centerPoint, "");
}
else
{
//Likely session timeout - return a Request Timeout error (408)...
string message = "User session has expired!!";
Response.StatusCode = (int)HttpStatusCode.RequestTimeout;
Response.StatusDescription = message;
Response.TrySkipIisCustomErrors = true; //Tell IIS to use your error text not the 'standard' error text!!
return(Json(new { Message = message }, "application/json"));
}
}
return(Json(markerjSON));
}