static void AggregeteFolder(string folderPath, string fileName, string errorsFileName)
{
ProgressCounter progress = new ProgressCounter();
List <AppData> apps = new List <AppData>();
List <Exception> exceptions = new List <Exception>();
foreach (string app_file_name in Directory.EnumerateFiles(folderPath, "*.htm"))
{
progress.LogEvery(10);
progress.Increment();
try
{
HtmlDocument document = new HtmlDocument();
document.LoadHtml(File.ReadAllText(app_file_name));
AppData appData = new AppData(document, "");
apps.Add(appData);
}
catch (Exception e)
{
e.Data["path"] = app_file_name;
exceptions.Add(e);
Console.WriteLine("errors: {0}", exceptions.Count());
}
}
MultithreadedCrawler.SaveExceptions(errorsFileName, exceptions);
AppData.save_to_file(apps, fileName);
}
protected override List <string> ProcessUrlAndGetLinks(WebClient webClient, string current_url, int processId)
{
//if (progress.Counter > 1000)
//{
// return new List<string>();
//}
try
{
if (progress.Counter % 10 == 0)
{
Console.WriteLine("progress.Counter = {0}, progress.CountPerSecond = {1}",
progress.Counter, progress.CountPerSecond());
}
progress.Increment();
//Console.WriteLine("\n({0}, {1}) ID={3} download {2}", visited_urls.Count, urls_to_visit.Count, current_url, processId);
string html = webClient.DownloadString(current_url);
HtmlDocument document = new HtmlDocument();
document.LoadHtml(html);
HtmlNode root = document.DocumentNode;
List <string> links =
root.SelectNodes("//a[@href]")
.Select(el => normalize_url(current_url, el.GetAttributeValue("href", null)))
.ToList();
List <string> filtered_links = links.Where(good_url).ToList();
IAppDataUS app = null;
try // parse app
{
Console.WriteLine(current_url);
app = new IAppDataUS(document, current_url); //\\
lock (apps)
{
apps.Add(app);
}
//Console.WriteLine(app.name + " " + app.url);
}
catch (Exception e)
{
AddException(e, current_url);
}
save_page_to_file(root, html, app);
return(filtered_links);
}
catch (Exception e)
{
AddException(e, current_url);
return(new List <string>());
}
}
public ContourAnimationInfo getContourAnimations(ContourQuery contourQuery)
{
List<List<TrendingDataLocation>> frames = GetFramesFromHistorian(contourQuery);
PiecewiseLinearFunction colorScale = GetColorScale(contourQuery);
Func<double, double> colorFunc = colorScale;
// The actual startDate is the timestamp of the
// first frame after contourQuery.GetStartDate()
DateTime startDate = contourQuery.GetStartDate();
int stepSize = contourQuery.StepSize;
int startTimeOffset = (int)Math.Ceiling((startDate - startDate.Date).TotalMinutes / stepSize);
startDate = startDate.Date.AddMinutes(startTimeOffset * stepSize);
double minLat = frames.Min(frame => frame.Min(location => location.Latitude)) - GetLatFromMiles(50.0D);
double maxLat = frames.Min(frame => frame.Max(location => location.Latitude)) + GetLatFromMiles(50.0D);
double minLng = frames.Min(frame => frame.Min(location => location.Longitude)) - GetLngFromMiles(50.0D, 0.0D);
double maxLng = frames.Min(frame => frame.Max(location => location.Longitude)) + GetLngFromMiles(50.0D, 0.0D);
GeoCoordinate topLeft = new GeoCoordinate(maxLat, minLng);
GeoCoordinate bottomRight = new GeoCoordinate(minLat, maxLng);
GSF.Drawing.Point topLeftPoint = s_crs.Translate(topLeft, contourQuery.Resolution);
GSF.Drawing.Point bottomRightPoint = s_crs.Translate(bottomRight, contourQuery.Resolution);
topLeftPoint = new GSF.Drawing.Point(Math.Floor(topLeftPoint.X), Math.Floor(topLeftPoint.Y));
bottomRightPoint = new GSF.Drawing.Point(Math.Ceiling(bottomRightPoint.X), Math.Ceiling(bottomRightPoint.Y));
topLeft = s_crs.Translate(topLeftPoint, contourQuery.Resolution);
bottomRight = s_crs.Translate(bottomRightPoint, contourQuery.Resolution);
int width = (int)(bottomRightPoint.X - topLeftPoint.X + 1);
int height = (int)(bottomRightPoint.Y - topLeftPoint.Y + 1);
int animationID;
string timeZoneID = null;
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("INSERT INTO ContourAnimation(ColorScaleName, StartTime, EndTime, StepSize) VALUES({0}, {1}, {2}, {3})", contourQuery.ColorScaleName, contourQuery.GetStartDate(), contourQuery.GetEndDate(), contourQuery.StepSize);
animationID = connection.ExecuteScalar<int>("SELECT @@IDENTITY");
if (contourQuery.IncludeWeather)
timeZoneID = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'XDATimeZone'");
}
GSF.Threading.CancellationToken cancellationToken = new GSF.Threading.CancellationToken();
s_cancellationTokens[animationID] = cancellationToken;
ProgressCounter progressCounter = new ProgressCounter(frames.Count);
s_progressCounters[animationID] = progressCounter;
Action<int> createFrame = i =>
{
List<TrendingDataLocation> frame = frames[i];
IDWFunc idwFunction = GetIDWFunction(contourQuery, frame);
uint[] pixelData;
if (contourQuery.IncludeWeather)
{
TimeZoneInfo tzInfo = !string.IsNullOrEmpty(timeZoneID)
? TimeZoneInfo.FindSystemTimeZoneById(timeZoneID)
: TimeZoneInfo.Local;
// Weather data is only available in 5-minute increments
DateTime frameTime = TimeZoneInfo.ConvertTimeToUtc(startDate.AddMinutes(stepSize * i), tzInfo);
double minutes = (frameTime - frameTime.Date).TotalMinutes;
int weatherMinutes = (int)Math.Ceiling(minutes / 5) * 5;
NameValueCollection queryString = HttpUtility.ParseQueryString(string.Empty);
queryString["service"] = "WMS";
queryString["request"] = "GetMap";
queryString["layers"] = "nexrad-n0r-wmst";
queryString["format"] = "image/png";
queryString["transparent"] = "true";
queryString["version"] = "1.1.1";
queryString["time"] = frameTime.Date.AddMinutes(weatherMinutes).ToString("o");
queryString["height"] = height.ToString();
queryString["width"] = width.ToString();
queryString["srs"] = "EPSG:3857";
GSF.Drawing.Point topLeftProjected = s_crs.Projection.Project(topLeft);
GSF.Drawing.Point bottomRightProjected = s_crs.Projection.Project(bottomRight);
queryString["bbox"] = string.Join(",", topLeftProjected.X, bottomRightProjected.Y, bottomRightProjected.X, topLeftProjected.Y);
string weatherURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/nexrad/n0r-t.cgi?" + queryString.ToString();
using (WebClient client = new WebClient())
using (MemoryStream stream = new MemoryStream(client.DownloadData(weatherURL)))
using (Bitmap bitmap = new Bitmap(stream))
{
pixelData = bitmap.ToPixelData();
}
}
else
{
pixelData = new uint[width * height];
}
if (cancellationToken.IsCancelled)
return;
for (int x = 0; x < width; x++)
{
if (cancellationToken.IsCancelled)
return;
for (int y = 0; y < height; y++)
{
if (cancellationToken.IsCancelled)
return;
if (pixelData[y * width + x] > 0)
continue;
GSF.Drawing.Point offsetPixel = new GSF.Drawing.Point(topLeftPoint.X + x, topLeftPoint.Y + y);
GeoCoordinate pixelCoordinate = s_crs.Translate(offsetPixel, contourQuery.Resolution);
double interpolatedValue = idwFunction(pixelCoordinate.Longitude, pixelCoordinate.Latitude);
pixelData[y * width + x] = (uint)colorFunc(interpolatedValue);
}
}
if (cancellationToken.IsCancelled)
return;
using (Bitmap bitmap = BitmapExtensions.FromPixelData(width, pixelData))
using (MemoryStream stream = new MemoryStream())
{
bitmap.Save(stream, ImageFormat.Png);
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("INSERT INTO ContourAnimationFrame VALUES({0}, {1}, {2})", animationID, i, stream.ToArray());
}
}
progressCounter.Increment();
};
Task.Run(() =>
{
ICancellationToken token;
ProgressCounter counter;
Parallel.For(0, frames.Count, createFrame);
s_cancellationTokens.TryRemove(animationID, out token);
s_progressCounters.TryRemove(animationID, out counter);
if (cancellationToken.IsCancelled)
{
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("DELETE FROM ContourAnimationFrame WHERE ContourAnimationID = {0}", animationID);
connection.ExecuteNonQuery("DELETE FROM ContourAnimation WHERE ID = {0}", animationID);
}
}
});
s_cleanUpAnimationOperation.TryRunOnceAsync();
return new ContourAnimationInfo()
{
AnimationID = animationID,
ColorDomain = colorScale.Domain,
ColorRange = colorScale.Range,
MinLatitude = bottomRight.Latitude,
MaxLatitude = topLeft.Latitude,
MinLongitude = topLeft.Longitude,
MaxLongitude = bottomRight.Longitude,
Infos = frames.Select((frame, index) => new ContourInfo()
{
Locations = frame,
URL = string.Format("./mapService.asmx/getContourAnimationFrame?animation={0}&frame={1}", animationID, index),
Date = contourQuery.GetStartDate().AddMinutes(index * contourQuery.StepSize).ToString()
}).ToList()
};
}
public ContourAnimationInfo getContourAnimations(ContourQuery contourQuery)
{
List <List <TrendingDataLocation> > frames = GetFramesFromHistorian(contourQuery);
PiecewiseLinearFunction colorScale = GetColorScale(contourQuery);
Func <double, double> colorFunc = colorScale;
// The actual startDate is the timestamp of the
// first frame after contourQuery.GetStartDate()
DateTime startDate = contourQuery.GetStartDate();
int stepSize = contourQuery.StepSize;
int startTimeOffset = (int)Math.Ceiling((startDate - startDate.Date).TotalMinutes / stepSize);
startDate = startDate.Date.AddMinutes(startTimeOffset * stepSize);
double latDif = frames.Min(frame => frame.Max(location => location.Latitude)) - frames.Min(frame => frame.Min(location => location.Latitude));
double lonDif = frames.Min(frame => frame.Max(location => location.Longitude)) - frames.Min(frame => frame.Min(location => location.Longitude));
double minLat = frames.Min(frame => frame.Min(location => location.Latitude)) - (latDif * 0.1D);
double maxLat = frames.Min(frame => frame.Max(location => location.Latitude)) + (latDif * 0.1D);
double minLng = frames.Min(frame => frame.Min(location => location.Longitude)) - (lonDif * 0.1D);
double maxLng = frames.Min(frame => frame.Max(location => location.Longitude)) + (lonDif * 0.1D);
GeoCoordinate topLeft = new GeoCoordinate(maxLat, minLng);
GeoCoordinate bottomRight = new GeoCoordinate(minLat, maxLng);
GSF.Drawing.Point topLeftPoint = s_crs.Translate(topLeft, contourQuery.Resolution);
GSF.Drawing.Point bottomRightPoint = s_crs.Translate(bottomRight, contourQuery.Resolution);
topLeftPoint = new GSF.Drawing.Point(Math.Floor(topLeftPoint.X), Math.Floor(topLeftPoint.Y));
bottomRightPoint = new GSF.Drawing.Point(Math.Ceiling(bottomRightPoint.X), Math.Ceiling(bottomRightPoint.Y));
topLeft = s_crs.Translate(topLeftPoint, contourQuery.Resolution);
bottomRight = s_crs.Translate(bottomRightPoint, contourQuery.Resolution);
int width = (int)(bottomRightPoint.X - topLeftPoint.X + 1);
int height = (int)(bottomRightPoint.Y - topLeftPoint.Y + 1);
int animationID;
string timeZoneID = null;
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("INSERT INTO ContourAnimation(ColorScaleName, StartTime, EndTime, StepSize) VALUES({0}, {1}, {2}, {3})", contourQuery.ColorScaleName, contourQuery.GetStartDate(), contourQuery.GetEndDate(), contourQuery.StepSize);
animationID = connection.ExecuteScalar <int>("SELECT @@IDENTITY");
if (contourQuery.IncludeWeather)
{
timeZoneID = connection.ExecuteScalar <string>("SELECT Value FROM Setting WHERE Name = 'XDATimeZone'");
}
}
GSF.Threading.CancellationToken cancellationToken = new GSF.Threading.CancellationToken();
s_cancellationTokens[animationID] = cancellationToken;
ProgressCounter progressCounter = new ProgressCounter(frames.Count);
s_progressCounters[animationID] = progressCounter;
Action <int> createFrame = i =>
{
List <TrendingDataLocation> frame = frames[i];
IDWFunc idwFunction = GetIDWFunction(contourQuery, frame);
uint[] pixelData;
if (contourQuery.IncludeWeather)
{
TimeZoneInfo tzInfo = !string.IsNullOrEmpty(timeZoneID)
? TimeZoneInfo.FindSystemTimeZoneById(timeZoneID)
: TimeZoneInfo.Local;
// Weather data is only available in 5-minute increments
DateTime frameTime = TimeZoneInfo.ConvertTimeToUtc(startDate.AddMinutes(stepSize * i), tzInfo);
double minutes = (frameTime - frameTime.Date).TotalMinutes;
int weatherMinutes = (int)Math.Ceiling(minutes / 5) * 5;
NameValueCollection queryString = HttpUtility.ParseQueryString(string.Empty);
queryString["service"] = "WMS";
queryString["request"] = "GetMap";
queryString["layers"] = "nexrad-n0r-wmst";
queryString["format"] = "image/png";
queryString["transparent"] = "true";
queryString["version"] = "1.1.1";
queryString["time"] = frameTime.Date.AddMinutes(weatherMinutes).ToString("o");
queryString["height"] = height.ToString();
queryString["width"] = width.ToString();
queryString["srs"] = "EPSG:3857";
GSF.Drawing.Point topLeftProjected = s_crs.Projection.Project(topLeft);
GSF.Drawing.Point bottomRightProjected = s_crs.Projection.Project(bottomRight);
queryString["bbox"] = string.Join(",", topLeftProjected.X, bottomRightProjected.Y, bottomRightProjected.X, topLeftProjected.Y);
string weatherURL = "http://mesonet.agron.iastate.edu/cgi-bin/wms/nexrad/n0r-t.cgi?" + queryString.ToString();
using (WebClient client = new WebClient())
using (MemoryStream stream = new MemoryStream(client.DownloadData(weatherURL)))
using (Bitmap bitmap = new Bitmap(stream))
{
pixelData = bitmap.ToPixelData();
}
}
else
{
pixelData = new uint[width * height];
}
if (cancellationToken.IsCancelled)
{
return;
}
for (int x = 0; x < width; x++)
{
if (cancellationToken.IsCancelled)
{
return;
}
for (int y = 0; y < height; y++)
{
if (cancellationToken.IsCancelled)
{
return;
}
if (pixelData[y * width + x] > 0)
{
continue;
}
GSF.Drawing.Point offsetPixel = new GSF.Drawing.Point(topLeftPoint.X + x, topLeftPoint.Y + y);
GeoCoordinate pixelCoordinate = s_crs.Translate(offsetPixel, contourQuery.Resolution);
double interpolatedValue = idwFunction(pixelCoordinate.Longitude, pixelCoordinate.Latitude);
pixelData[y * width + x] = (uint)colorFunc(interpolatedValue);
}
}
if (cancellationToken.IsCancelled)
{
return;
}
using (Bitmap bitmap = BitmapExtensions.FromPixelData(width, pixelData))
using (MemoryStream stream = new MemoryStream())
{
bitmap.Save(stream, ImageFormat.Png);
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("INSERT INTO ContourAnimationFrame VALUES({0}, {1}, {2})", animationID, i, stream.ToArray());
}
}
progressCounter.Increment();
};
Task.Run(() =>
{
ICancellationToken token;
ProgressCounter counter;
Parallel.For(0, frames.Count, createFrame);
s_cancellationTokens.TryRemove(animationID, out token);
s_progressCounters.TryRemove(animationID, out counter);
if (cancellationToken.IsCancelled)
{
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
connection.ExecuteNonQuery("DELETE FROM ContourAnimationFrame WHERE ContourAnimationID = {0}", animationID);
connection.ExecuteNonQuery("DELETE FROM ContourAnimation WHERE ID = {0}", animationID);
}
}
});
s_cleanUpAnimationOperation.TryRunOnceAsync();
return(new ContourAnimationInfo()
{
AnimationID = animationID,
ColorDomain = colorScale.Domain,
ColorRange = colorScale.Range,
MinLatitude = bottomRight.Latitude,
MaxLatitude = topLeft.Latitude,
MinLongitude = topLeft.Longitude,
MaxLongitude = bottomRight.Longitude,
Infos = frames.Select((frame, index) => new ContourInfo()
{
Locations = frame,
URL = string.Format("./mapService.asmx/getContourAnimationFrame?animation={0}&frame={1}", animationID, index),
Date = contourQuery.GetStartDate().AddMinutes(index * contourQuery.StepSize).ToString()
}).ToList()
});
}
protected override List <string> ProcessUrlAndGetLinks(WebClient webClient, string current_url, int processId)
{
progress.Increment();
progress.LogEvery(100);
//Console.WriteLine("\n({0}, {1}) ID={3} download {2}", visited_urls.Count, urls_to_visit.Count, current_url, processId);
string stage = null;
string html = null;
List <string> filtered_links = null;
try
{
stage = "downloading";
html = webClient.DownloadString(current_url);
HtmlDocument document = new HtmlDocument();
stage = "loading_html_document";
document.LoadHtml(html);
HtmlNode root = document.DocumentNode;
stage = "get_links";
List <string> links =
root.SelectNodes("//a[@href]")
.Select(el => normalize_url(current_url, el.GetAttributeValue("href", null)))
.ToList();
filtered_links = links.Where(good_url).ToList();
if (AppData.IsAppUrl(current_url))
{
stage = "parse_app";
AppData app = new AppData(document, current_url);
lock (apps)
{
apps.Add(app);
}
SavePage(_appsPath, html, app.Id);
stage = "parse_youtube_page";
ProcessVideoId(webClient, app.videoID);
}
else if (DeveloperData.IsDeveloperUrl(current_url))
{
stage = "parse_developer";
DeveloperData developer = new DeveloperData(document, current_url);
lock (developers)
{
developers.Add(developer);
}
SavePage(_developersPath, html, developer.name);
}
else
{
SavePage(_otherPagesPath, html);
}
}
catch (Exception e)
{
AddException(e, current_url, stage);
SaveFaultyPage(html, stage, e);
}
return(filtered_links ?? new List <string>());
}
