public static string GetHistory(string weatherStationDeviceId) { try { var airportCode = RetrieveNearestAirport(weatherStationDeviceId); Logger.LocalLog("WeatherUtility", "GetHistory", "LogMessage", "Adding to downloadlist " + airportCode); if (!string.IsNullOrWhiteSpace(airportCode)) { var obj = new AirportDownloadModel { AirportCode = airportCode, Downloaded = false, WeatherStationDeviceId = weatherStationDeviceId, EndDownload = GetEarliestRecord(weatherStationDeviceId), LastDownload = new DateTime(DateTime.Now.Year - 1, 1, 1), StartDownload = new DateTime(DateTime.Now.Year - 1, 1, 1) }; TableStorageRestHelper.InsertOrReplaceEntity(Invariables.SwitchStorageConnection, Invariables.weatherHistoryTableName, Invariables.partitionKey, airportCode, obj); } return(airportCode); } catch (Exception exception) { Debug.WriteLine(exception.Message); Logger.ErrorLog("DataHelper", "GetHistory", exception.ToString( )); return(null); } }
private void PostDegreeDayValuesForPreviousDay(Guid deviceId, DateTime startDateTime, DateTime endDateTime) { try { string cddSensor = ""; string hddSensor = ""; string extTempSensor = ""; StringBuilder payload = new StringBuilder( ); // get cdd / hdd sensors of weatherstation var sql = @"SELECT ObjectPropertyId, PropertyName FROM ObjectProperties WHERE DeviceId = @device AND (PropertyName LIKE '%externaltemperature%' OR PropertyName LIKE '%degreeday%')" ; var dt = SqlHelper.ExecuteReader(Invariables.SwitchConnection, sql, new List <SqlParam> { new SqlParam("device", deviceId, SqlDbType.UniqueIdentifier) }); if (dt != null && dt.Rows.Count > 0) { foreach (DataRow nRow in dt.Rows) { switch (nRow["PropertyName"].ToString( ).ToLower( )) { case "externaltemperature": extTempSensor = nRow["ObjectPropertyId"].ToString( ); break; case "coolingdegreeday": cddSensor = nRow["ObjectPropertyId"].ToString( ); break; case "heatingdegreeday": hddSensor = nRow["ObjectPropertyId"].ToString( ); break; } } } string rowFilter = $"PartitionKey eq '{Invariables.wwoInstallation}' and RowKey ge '{extTempSensor}:{startDateTime.Ticks}' and RowKey le '{extTempSensor}:{endDateTime.Ticks}'"; var readings = TableStorageRestHelper.QueryEntities <WeatherReading>(Invariables.SwitchStorageConnection, "ObservationStaging", rowFilter); decimal totalHdd = 0; decimal totalCdd = 0; decimal partOfDay = decimal.Parse("1") / decimal.Parse(readings.Count.ToString( )); foreach (WeatherReading reading in readings) { decimal heating, cooling; decimal baseTemperature = decimal.Parse("18"); heating = (baseTemperature - decimal.Parse(reading.Value.ToString(CultureInfo.InvariantCulture))) * partOfDay; cooling = (decimal.Parse(reading.Value.ToString(CultureInfo.InvariantCulture)) - baseTemperature) * partOfDay; decimal hdd = (heating) < 0 ? 0 : heating; decimal cdd = (cooling) < 0 ? 0 : cooling; totalHdd = totalHdd + hdd; totalCdd = totalCdd + cdd; } payload.Append("["); // heating degree day payload.Append(DataHelper.GeneratePayloadEntry(payload, hddSensor, endDateTime.ToString("yyyy-MM-ddT16:00:00Z"), Math.Round(totalHdd, 4).ToString(CultureInfo.InvariantCulture))); // cooling degree day payload.Append(DataHelper.GeneratePayloadEntry(payload, cddSensor, endDateTime.ToString("yyyy-MM-ddT16:00:00Z"), Math.Round(totalCdd, 4).ToString(CultureInfo.InvariantCulture))); payload.Append("]"); // Post CDD & HDD DataHelper.PostObservationEnqueue(_projectId, Invariables.wwoInstallation, payload.ToString( ), _apikey); } catch (Exception ex) { progressStatusText.Text = ex.Message; progressDownloadBar.Value = 10; Logger.LocalLog("WeatherUtility", "PostDegreeDayValuesForPreviousDay", "LogMessage", ex.ToString( )); } }
private void DownloadAirportHistorical(List <AirportDownloadModel> airportDownload) { try { Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", "Processing download list..."); var web = new WebClient(); foreach (var apdl in airportDownload) { if ((apdl.EndDownload - apdl.StartDownload).TotalDays > 365) { Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", "Download 1 year only " + apdl.StartDownload.ToString("yyyy-MM-dd") + " to " + apdl.EndDownload.ToString("yyyy-MM-dd")); apdl.StartDownload = apdl.EndDownload.AddDays(-365); } var current = apdl.StartDownload; var weatherStation = DataHelper.GetStation(apdl.WeatherStationDeviceId); if (weatherStation == null) { return; } var payload = new StringBuilder(); var currentTime = DateTime.Now.Date; Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", $"Downloading airport : {apdl.AirportCode} from {apdl.StartDownload} to {apdl.EndDownload}"); while (current <= apdl.EndDownload) { try { // compute hdd and cdd before adding up a day int offset = -1; TimeHelper.GetUtcOffsetForLocalDatetimeForInstallation( Guid.Parse(Invariables.wwoInstallation), current.Date, out offset); offset = (-1 * offset); var utcStart = DateTime.SpecifyKind(current.Date.AddMinutes(offset), DateTimeKind.Utc); var utcEnd = DateTime.SpecifyKind( current.Date.AddDays(1).AddSeconds(-1).AddMinutes(offset), DateTimeKind.Utc); PostDegreeDayValuesForPreviousDay(weatherStation.DeviceId, utcStart, utcEnd); //string[] lines = null; //Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", // DateTime.Now + " Downloading airport : " + apdl.AirportCode + " Date " + current); //var result = web.DownloadString("http://www.wunderground.com/history/airport/" + // apdl.AirportCode + "/" + current.Year + "/" + current.Month + // "/" + current.Day + // "/DailyHistory.html?req_city=NA&req_state=NA&req_statename=NA&format=1"); //File.AppendAllText(@"C:\Logs\WUndergroundHistoricalData.csv", result); //var fn = apdl.AirportCode + "/" + apdl.AirportCode + "_" + current.ToString("yyyyMMdd") + ".csv"; //result = result.Replace("\n", ""); //result = result.Replace("<br />", "\r"); //lines = result.Split('\r'); //// retreive the weatherstations //var obj = new AirportDownloadModel //{ // AirportCode = apdl.AirportCode, // Downloaded = false, // EndDownload = apdl.EndDownload, // StartDownload = current, // WeatherStationDeviceId = apdl.WeatherStationDeviceId //}; //var doUpload = true; //var convertToCelcius = false; //// Check missing interval readings only //// Get data of the missing interval readings //var offset = 0; //TimeHelper.GetUtcOffsetForLocalDatetimeForInstallation( // Guid.Parse(Invariables.wwoInstallation), current.Date, out offset); //offset = (-1*offset); //var utcStart = DateTime.SpecifyKind(current.Date.AddMinutes(offset), DateTimeKind.Utc); //var utcEnd = DateTime.SpecifyKind( // current.Date.AddDays(1).AddSeconds(-1).AddMinutes(offset), DateTimeKind.Utc); //var errormessage = ""; //var sensorList = DataHelper.GetFirstSensor(Guid.Parse(apdl.WeatherStationDeviceId), // out errormessage); //var getReadings = new List<WeatherReading>(); //foreach (var sensor in sensorList) //{ // var rowfilter = // $"PartitionKey eq '{Invariables.wwoInstallation}' and RowKey ge '{sensor}:{(utcStart.AddMinutes(-15)).Ticks}' and RowKey le '{sensor}:{utcEnd.Ticks}'"; // getReadings.AddRange( // TableStorageRestHelper.QueryEntities<WeatherReading>( // Invariables.SwitchStorageConnection, Invariables.readingsTableName, rowfilter, 500000)); //} //// delete interval reading record //var c = 0; //var isDeleted = false; //while (!isDeleted & c < 5) //{ // c++; // isDeleted = TableStorageRestHelper.DeleteEntities(Invariables.SwitchStorageConnection, // Invariables.readingsTableName, Invariables.wwoInstallation, // getReadings.Cast<object>().ToList()); //} //for (var i = 0; i < lines.Length; i++) //{ // try // { // if (!string.IsNullOrWhiteSpace(lines[i])) // { // if (i == 0) // { // if (!lines[i].ToLower().Contains("airport code,")) // lines[i] = "Airport Code," + lines[i]; // var tokens2 = DataHelper.StringSplit(lines[i]); // if (tokens2[2].ToLower() == "temperaturef") // convertToCelcius = true; // } // else // { // var isMissing = false; // var tokens = DataHelper.StringSplit(lines[i]); // if (tokens[0] != apdl.AirportCode) // { // lines[i] = apdl.AirportCode + "," + lines[i]; // tokens = DataHelper.StringSplit(lines[i]); // } // DateTime.TryParse(tokens[14], out currentTime); // currentTime = DateTime.SpecifyKind(currentTime, DateTimeKind.Utc); // var currentTimeString = // (currentTime.ToUniversalTime()).ToUniversalTime() // .ToString("yyyy-MM-ddTHH:00:00Z"); // isMissing = true; // double externalTemperature; // double.TryParse(tokens[2], out externalTemperature); // if (convertToCelcius) // externalTemperature = Math.Round((externalTemperature - 32)*0.5555, 2); // double humidity; // double.TryParse(tokens[4], out humidity); // double pressure; // double.TryParse(tokens[5], out pressure); // double visibility; // double.TryParse(tokens[6], out visibility); // double winddirection; // double.TryParse(tokens[7], out winddirection); // double windspeed; // double.TryParse(tokens[8], out windspeed); // double currentRain; // double.TryParse(tokens[10], out currentRain); // if (isMissing) // { // payload.Clear().Append("["); // // we need to filter external temperature that is <200 // // coming from wunderground // // external temperature // if (externalTemperature > -200 && externalTemperature < 400) // { // if ( // weatherStation.StationProperties.ContainsKey( // "ExternalTemperature")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["ExternalTemperature"], // currentTimeString, externalTemperature.ToString("N2"))); // } // // humidity // if (weatherStation.StationProperties.ContainsKey("Humidity")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["Humidity"], // currentTimeString, humidity.ToString("N2"))); // // windspeed // if (weatherStation.StationProperties.ContainsKey("WindSpeed")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["WindSpeed"], // currentTimeString, windspeed.ToString("N2"))); // // wind direction // if (weatherStation.StationProperties.ContainsKey("WindDirection")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["WindDirection"], // currentTimeString, winddirection.ToString("N2"))); // // barometric pressure // if (weatherStation.StationProperties.ContainsKey("BarometricPressure")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["BarometricPressure"], // currentTimeString, pressure.ToString("N2"))); // // visibility // if (weatherStation.StationProperties.ContainsKey("Visibility")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["Visibility"], // currentTimeString, visibility.ToString("N2"))); // // compute the wet bulb // if (weatherStation.StationProperties.ContainsKey("WetBulbTemperature")) // { // var tempc = externalTemperature; // var rh = humidity; // var wetbulbTemperatureF = // DataHelper.ComputeWetBulbTemperature( // DataHelper.CelciusToF(tempc), rh); // if (!double.IsNaN(wetbulbTemperatureF)) // { // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["WetBulbTemperature"], // currentTimeString, // DataHelper.FahrenheitToC(wetbulbTemperatureF).ToString("N2"))); // } // } // // rain & current rain // double sample = 0; // currentRain = Math.Round(currentRain, 2); // if (weatherStation.StationProperties.ContainsKey("RainCurrent")) // { // var prevrain = weatherStation.CurrentRain; // if (prevrain > currentRain) // sample = currentRain; // else // sample = currentRain - prevrain; // if (weatherStation.StationProperties.ContainsKey("RainCurrent")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["RainCurrent"], // currentTimeString, currentRain.ToString("N2"))); // if (weatherStation.StationProperties.ContainsKey("Rain")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["Rain"], // currentTimeString, sample.ToString("N2"))); // weatherStation.CurrentRain = currentRain; // } // else if (weatherStation.StationProperties.ContainsKey("Rain")) // payload.Append(DataHelper.GeneratePayloadEntry(payload, // weatherStation.StationProperties["Rain"], // currentTimeString, sample.ToString("N2"))); // payload.Append("]"); // DataHelper.PostObservationEnqueue(_projectId, // Invariables.wwoInstallation, payload.ToString(), _apikey); // } // } // } // } // catch (Exception) // { // } // if (i%10 == 0) // Thread.Sleep(10); //} //if (doUpload) //{ // //Logger.LocalLog( "WeatherUtility", "DownloadAirportHistorical", "LogMessage", "Uploading to container [" + Invariables.weatherHistory + "] file: " + fn ); // File.WriteAllLines(@"C:\temp\temp.tmp", lines); // DataHelper.Upload(new MemoryStream(File.ReadAllBytes(@"C:\temp\temp.tmp")), fn, // Invariables.weatherHistory); //} //// update the entry //obj.AirportCode = apdl.AirportCode; //obj.Downloaded = false; //obj.EndDownload = apdl.EndDownload; //obj.StartDownload = current; //obj.WeatherStationDeviceId = weatherStation.DeviceId.ToString(); //var retry = 0; //var ok = false; //while (!ok && retry < 5) //{ // retry++; // Thread.Sleep(5); // ok = TableStorageRestHelper.InsertOrReplaceEntity(Invariables.SwitchStorageConnection, // Invariables.weatherHistoryTableName, Invariables.partitionKey, apdl.AirportCode, obj); //} } catch (Exception ex) { Logger.ErrorLog("WeatherUtility", "ProcessNewInstallation", ex.ToString()); } finally { current = current.AddDays(1); Thread.Sleep(10); } } if (current >= apdl.EndDownload) { TableStorageRestHelper.DeleteEntity(Invariables.SwitchStorageConnection, Invariables.weatherHistoryTableName, Invariables.partitionKey, apdl.AirportCode); Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", $"Done downloading {apdl.AirportCode}."); } } } catch (Exception ex) { progressStatusText.Text = ex.Message; progressDownloadBar.Value = 10; Logger.LocalLog("WeatherUtility", "DownloadAirportHistorical", "LogMessage", ex.ToString()); } }