//data.taipe 目錄抓取
public void craw()
{
int count = 0;
int offset = 0; //跳過x筆資料
int limit = 100; //回傳數量限制
int first = 0;
WebClient client = new WebClient();
if (File.Exists(System.Windows.Forms.Application.StartupPath + @"\metadata.txt"))
{
StreamReader sr1 = new StreamReader(System.Windows.Forms.Application.StartupPath + @"\metadata.txt", Encoding.UTF8);
while (!sr1.EndOfStream)
{
Data temp = new Data();
string[] data = sr1.ReadLine().Split(new string[] { "\t\t" }, StringSplitOptions.RemoveEmptyEntries);
temp.Name = data[0];
temp.ID = data[1];
if (data.Length == 3)
temp.RID = data[2];
allData.Add(temp);
}
}
else
{
StreamWriter sw1 = new StreamWriter(System.Windows.Forms.Application.StartupPath + @"\metadata.txt", false, Encoding.UTF8);
do
{
MemoryStream ms = new MemoryStream(client.DownloadData(string.Format("http://data.taipei/opendata/datalist/apiAccess?scope=datasetMetadataSearch&limit={0}&offset={1}", limit, offset)));
HtmlDocument doc = new HtmlDocument();
doc.Load(ms, Encoding.UTF8);
try
{
dynamic data = JsonConvert.DeserializeObject(doc.DocumentNode.InnerHtml);
count = data.result.count;
foreach (dynamic a in data.result.results)
{
Data temp = new Data();
temp.ID = a.id;
temp.Name = a.title;
try
{
if (a.resources.Count != 0)
{
if (a.resources[0].resourceId !=null)
temp.RID = a.resources[0].resourceId;
}
}
catch (Exception e)
{
temp.RID = a.resources.resourceId;
}
allData.Add(temp);
if (first == 0)
{
first++;
}
sw1.WriteLine(temp.Name + "\t\t" + temp.ID+ "\t\t" + temp.RID);
offset++;
}
}
catch (Exception e)
{
Console.WriteLine("Error:" + e.Message);
string contexnt = doc.DocumentNode.InnerHtml;
while (contexnt.Contains("\"id\":\""))
{
Data temp = new Data();
int str = 0, end = 0;
str = contexnt.IndexOf("\"id\":\"");
end = contexnt.IndexOf("\",\"title\":\"");
temp.ID = contexnt.Substring(str + 6, end - (str + 6));
contexnt = contexnt.Substring(end + 11);
end = contexnt.IndexOf("\",\"");
temp.Name = contexnt.Substring(0, end);
contexnt = contexnt.Substring(end);
str = end;
if (contexnt.IndexOf("\"id\":\"") != -1)
{
end = contexnt.IndexOf("\"id\":\"");
string res = contexnt.Substring(str, end - str);
if (res.Contains("\"resourceId\":\""))
{
str = contexnt.IndexOf("\"resourceId\":\"");
contexnt = contexnt.Substring(str + 14);
end = contexnt.IndexOf("\",\"");
temp.RID = contexnt.Substring(0, end);
contexnt = contexnt.Substring(end);
}
}
allData.Add(temp);
sw1.WriteLine(temp.Name + "\t\t" + temp.ID + "\t\t" + temp.RID);
offset++;
}
}
} while ((count-offset) > 0);
sw1.Close();
}
Console.WriteLine("sdfsadf");
// _form.update("ffff");
_form.update(allData);
}
public void weatherCraw()
{
WebClient client = new WebClient();
if (File.Exists(System.Windows.Forms.Application.StartupPath + @"\weathercontent.txt"))
{
StreamReader sr1 = new StreamReader(System.Windows.Forms.Application.StartupPath + @"\weathercontent.txt", Encoding.UTF8);
// StreamReader sr2 = new StreamReader(System.Windows.Forms.Application.StartupPath + @"\metadata_id.txt", Encoding.UTF8);
while (!sr1.EndOfStream)
{
Data temp = new Data();
string[] data = sr1.ReadLine().Split(new string[] { "\t\t"," ","\t"," " }, StringSplitOptions.RemoveEmptyEntries);
temp.weatherContent = data[0];
temp.weatherKey = data[1];
weatherData.Add(temp);
}
sr1.Close();
}
_form.updateWeather(weatherData);
}