public void insertDataLog(DataLakeLog datalog)
{
SqlConnection conn = new SqlConnection(connStr);
conn.Open();
SqlCommand cmd = new SqlCommand("SpInsertDataLakeLog", conn);
try
{
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.AddWithValue("@dataset", datalog.dataset);
cmd.Parameters.AddWithValue("@datsetUrl", datalog.datsetUrl);
cmd.Parameters.AddWithValue("@lastrundate", datalog.lastrundate);
cmd.Parameters.AddWithValue("@createddate", datalog.createddate);
cmd.Parameters.AddWithValue("@iscomplete", datalog.iscomplete);
cmd.Parameters.AddWithValue("@DataLakeLogDescription", datalog.DataLakeLogDescription);
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
throw ex;
}
}
static public void Tick(Object stateInfo)
{
//Console.WriteLine("Tick: {0}", DateTime.Now.ToString("h:mm:ss"));
string URL = @"http://www.ncdc.noaa.gov/cdo-web/";
Program program = new Program();
//////calling Data old ///////////
/*
* try
* {
* using (var client = new HttpClient())
* {
* client.BaseAddress = new Uri(URL);
* client.DefaultRequestHeaders.Accept.Clear();
* client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
* client.DefaultRequestHeaders.Add("Token", "MnvlFcXOTrQtlaAYCTghEYNQABKSMiKh");
* string startdate = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd");
* //string startdate = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd");
* string enddate = DateTime.Now.ToString("yyyy-MM-dd");
*
* HttpResponseMessage response = client.GetAsync("api/v2/data?datasetid=GHCND&locationid=ZIP:28801&startdate=" + startdate + "&enddate=" + enddate + "").Result;
* //HttpResponseMessage respons8 = client.GetAsync("api/v2/data?datasetid=GHCND&locationid=ZIP:28801&startdate=2010-05-01&enddate=2010-05-01&limit=50").Result;
* //HttpResponseMessage response1 = client.GetAsync("api/v2/data?datasetid=ANNUAL&locationid=ZIP:28802&startdate=2010-05-01&enddate=2010-05-01").Result;
* //HttpResponseMessage response2 = client.GetAsync("api/v2/data?datasetid=GHCNDL&locationid==ZIP:28801&startdate=2010-05-02&enddate=2010-05-02").Result;
* //HttpResponseMessage response3 = client.GetAsync("api/v2/data?datasetid=GHCNDL&locationid==ZIP:28802&startdate=2010-05-02&enddate=2010-05-02&limit=100").Result;
* //HttpResponseMessage response3 = client.GetAsync("api/v2/data?datasetid=PRECIP_15&stationid=COOP:010008&units=metric&startdate=2010-05-01&enddate=2010-05-31&limit=100").Result;
* //HttpResponseMessage response4 = client.GetAsync("api/v2/data?datasetid=GSOM&stationid=GHCND:USC00010008&units=standard&startdate=2010-05-01&enddate=2010-05-31&limit=100").Result;
* //&locationid=FIPS:38
* //No data found for below parameters /
* //HttpResponseMessage response5 = client.GetAsync("api/v2/data?datasetid=ANNUAL&locationid=FIPS:37&startdate=2010-05-02&enddate=2010-05-02").Result;
* //HttpResponseMessage response6 = client.GetAsync("api/v2/data?datasetid=ANNUAL&locationid=FIPS:38&startdate=2010-05-01&enddate=2010-05-01").Result;
*
* if (response.IsSuccessStatusCode)
* {
* var jsondata = response.Content.ReadAsStringAsync().Result;
* //var jsondata1 = response1.Content.ReadAsStringAsync().Result;
* //var jsondata2 = response2.Content.ReadAsStringAsync().Result;
* // var jsondata3 = response3.Content.ReadAsStringAsync().Result;
* //var jsondata4 = response4.Content.ReadAsStringAsync().Result;
* //var jsondata5 = response5.Content.ReadAsStringAsync().Result;
* //var jsondata6 = response4.Content.ReadAsStringAsync().Result;
*
* var alldata = (JObject)JsonConvert.DeserializeObject(jsondata);
*
* //var alldata = (JObject)JsonConvert.DeserializeObject(jsondata4);
* //var alldata2 = (JObject)JsonConvert.DeserializeObject(jsondata3);
* //var alldata1 = (JObject)JsonConvert.DeserializeObject(jsondata);
* //alldata.Merge(alldata2);
* //alldata.Merge(alldata1);
* //var data = (JObject)JsonConvert.DeserializeObject(jsondata4);
* var alldataresult = alldata["results"].Select(item => new
* ALLData
* {
* date = Convert.ToDateTime(item["date"]),
* datatype = item["datatype"].ToString(),
* stations = item["station"].ToString(),
* attributes = item["attributes"].ToString(),
* value = Convert.ToInt16(item["value"]),
* }).ToList();
* program.AllDataInsert(alldataresult);
* }
* Console.WriteLine("Data Updated on " + DateTime.Now);
* Console.WriteLine();
*
* }
* }
* catch (Exception ex)
* {
* throw ex;
* }
* ///////////// End Data old ////////////////
*/
//////calling Data ///////////
try
{
using (var client = new HttpClient())
{
client.BaseAddress = new Uri(URL);
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("Token", "MnvlFcXOTrQtlaAYCTghEYNQABKSMiKh");
//string startdate = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd");
string lastrundate1 = "";
DateTime date1 = Convert.ToDateTime(ConfigurationManager.AppSettings["startdatetime"]);//Convert.ToDateTime("18/07/2016");
string startdate = date1.ToString("yyyy-MM-ddTHH:mm:ss");
///string startdate = ConfigurationManager.AppSettings["startdatetime"] ;
//string startdate = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd");
//DateTime date2 = Convert.ToDateTime(ConfigurationManager.AppSettings["enddatetime"]);
string enddate = DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss");
//string enddate = date2.ToString("yyyy-MM-ddTHH:mm:ss");
DateTime isnulldate = Convert.ToDateTime("1900-01-01 00:00:00.000");
//DateTime lastrundate = new DateTime();
//lastrundate1 = DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss");
GetAllDataBAL lastdate = new GetAllDataBAL();
DateTime lastrundate = lastdate.getlastrundate();
lastrundate1 = lastrundate.ToString("yyyy-MM-ddTHH:mm:ss");
DataLakeLog datalog = new DataLakeLog();
string URL1;
if (lastrundate == isnulldate)
{
URL1 = "api/v2/data?datasetid=GHCND&startdate=" + startdate + "&enddate=" + enddate + "&limit=1000";
lastrundate1 = startdate;
//lastrundate = Convert.ToDateTime("07/20/2016THH:mm:ss"); //DateTime.Now;
//lastrundate1 = DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss");
}
else
{
//enddate = DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss");
URL1 = "api/v2/data?datasetid=GHCND&startdate=" + lastrundate1 + "&enddate=" + enddate + "&limit=1000";
//lastrundate1 = lastrundate.ToString();
}
string dataseturl = URL + URL1;
HttpResponseMessage response = client.GetAsync(URL1).Result;
if (response.IsSuccessStatusCode)
{
//int len = response.Content.ReadAsStringAsync().Result.Length;
string jsondata = response.Content.ReadAsStringAsync().Result;
var alldata = (JObject)JsonConvert.DeserializeObject(jsondata);
int cnt1 = alldata.Count;
if (cnt1 != 0)
{
int cnt = Convert.ToInt32(alldata["metadata"]["resultset"]["count"]);
for (int i = 1001; i < cnt; i = i + 1000)
{
string offset = i.ToString();
HttpResponseMessage response1;//
//string url1 = "api/v2/locations?offset=" + i + "&limit=1000";
string url1 = "api/v2/data?datasetid=GHCND&startdate=" + lastrundate1 + "&enddate=" + enddate + "&offset=" + i + "&limit=1000";
response1 = client.GetAsync(url1).Result;
if (response1.IsSuccessStatusCode)
{
var jsondata1 = response1.Content.ReadAsStringAsync().Result;
var datanfinal = (JObject)JsonConvert.DeserializeObject(jsondata1);
alldata.Merge(datanfinal);
jsondata1 = null;
//locations = null;
}
}
var alldataresult = alldata["results"].Select(item => new
ALLData
{
date = Convert.ToDateTime(item["date"]),
datatype = item["datatype"].ToString(),
stations = item["station"].ToString(),
attributes = item["attributes"].ToString(),
value = Convert.ToInt16(item["value"]),
}).ToList();
program.AllDataInsert(alldataresult);
datalog.dataset = "GHCND";
datalog.datsetUrl = dataseturl;
datalog.lastrundate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss")); //lastrundate;
datalog.createddate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss"));
datalog.iscomplete = true;
datalog.DataLakeLogDescription = response.ReasonPhrase;
DateLakeLogBAL lakebal = new DateLakeLogBAL();
lakebal.insertDatalog(datalog);
Console.WriteLine("Data Updated on " + DateTime.Now);
Console.WriteLine();
Console.ReadLine();
}
else
{
datalog.dataset = "GHCND";
datalog.datsetUrl = dataseturl;
datalog.lastrundate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss")); //lastrundate;
datalog.createddate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss"));
datalog.iscomplete = false;
datalog.DataLakeLogDescription = "No data found for given set of parameters";
DateLakeLogBAL lakebal = new DateLakeLogBAL();
lakebal.insertDatalog(datalog);
Console.WriteLine("No data found for given set of parameters " + enddate);
Console.ReadLine();
}
}
else
{
datalog.dataset = "GHCND";
datalog.datsetUrl = dataseturl;
datalog.lastrundate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss")); //lastrundate;
datalog.createddate = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss"));
datalog.iscomplete = false;
datalog.DataLakeLogDescription = response.ReasonPhrase;
DateLakeLogBAL lakebal = new DateLakeLogBAL();
lakebal.insertDatalog(datalog);
Console.WriteLine(" " + response.ReasonPhrase + " For given set of parameters " + enddate);
Console.ReadLine();
}
}
}
catch (Exception ex)
{
//throw ex;
Console.WriteLine("error" + ex);
Console.ReadLine();
}
///////////// End Data ////////////////
}
public void insertDatalog(DataLakeLog datalog)
{
DatalakeLogDAL datalogdal = new DatalakeLogDAL();
datalogdal.insertDataLog(datalog);
}
