private bool IsMd5FilterPass(string text, IFileInfo item) { if (string.IsNullOrEmpty(text)) { return(true); } if (!(item is FileInfoWrapper)) { return(true); } var md5 = Stuff.CalcMD5((item as FileInfoWrapper)); return(md5 == text); }
private void button3_Click(object sender, EventArgs e) { if (th == null) { button3.Text = "abort"; } else { button3.Text = "download all"; th.Abort(); th = null; return; } Infos.Clear(); savePic = true; //webBrowser1.ScriptErrorsSuppressed = true; int cnt = listView1.Items.Count; //calc md5 in target dir saveTarget = textBox1.Text; if (radioButton1.Checked) { var lib = (comboBox4.SelectedItem as ComboBoxItem).Tag as FilesystemLibrary; saveTarget = lib.BaseDirectory.FullName; if (checkBox6.Checked) { saveTarget = Path.Combine(saveTarget, textBox4.Text); } } var dir = new DirectoryInfo(saveTarget); if (!dir.Exists) { Stuff.Error("directory " + dir.FullName + " not exist"); return; } skipped = 0; saved = 0; links = 0; errors = 0; List <string> ss = new List <string>(); for (int i = 0; i < listView1.Items.Count; i++) { ss.Add((string)listView1.Items[i].Tag); } th = new Thread(() => { statusStrip1.Invoke((Action)(() => { toolStripStatusLabel1.Text = "calculating md5 hashes.."; toolStripProgressBar1.Value = 0; toolStripProgressBar1.Visible = true; })); var fls = dir.GetFiles().ToArray(); int fcnt = 0; foreach (var item in fls) { var md5 = Stuff.CalcMD5(new FileInfoWrapper(item.FullName)); md5cache.Add(md5); statusStrip1.Invoke((Action)(() => { toolStripStatusLabel1.Text = "md5 hash: " + fcnt + " / " + fls.Length; toolStripProgressBar1.Value = (int)((fcnt / (float)fls.Length) * 100); })); fcnt++; } statusStrip1.Invoke((Action)(() => { toolStripStatusLabel1.Text = "starting downloads.."; toolStripProgressBar1.Value = 100; toolStripProgressBar1.Visible = true; })); for (int i = 0; i < ss.Count; i++) { statusStrip1.Invoke((Action)(() => { toolStripProgressBar1.Visible = true; toolStripProgressBar1.Value = (int)Math.Round(((float)i / (ss.Count)) * 100f); })); try { if (useDelay) { Thread.Sleep(delay); } var item = ss[i]; // iscomplete = false; //using (WebClient wc = new WebClient()) { // AppendHeaders(wc); // var str = wc.DownloadString(item); var req = HttpWebRequest.Create(item) as HttpWebRequest; AppendHeaders(req); req.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; var resp = req.GetResponse() as HttpWebResponse; if (resp.StatusCode != HttpStatusCode.OK) { } var enc = Encoding.GetEncoding(resp.CharacterSet); var strm = resp.GetResponseStream(); StreamReader readStream = new StreamReader(strm, enc); var str = readStream.ReadToEnd(); if (savePic) { savePicFunc(item, str); } //webBrowser1.Navigate((string)(item as ListViewItem).Tag, null, null, "User-Agent: Mozilla/5.0"); statusStrip1.Invoke((Action)(() => { toolStripStatusLabel1.Text = "skipped: " + skipped + " saved: " + saved + "; errors: " + errors + "; links extracted: " + links; })); } } catch (Exception ex) { errors++; } } statusStrip1.Invoke((Action)(() => { toolStripStatusLabel1.Text = "skipped: " + skipped + " saved: " + saved + "; errors: " + errors + "; links extracted: " + links; toolStripProgressBar1.Value = 100; toolStripProgressBar1.Visible = false; })); if (checkBox3.Checked)//close on complete { Application.Exit(); } button3.Invoke((Action)(() => { button3.Text = "download all"; })); th = null; }); th.IsBackground = true; th.Start(); }
public void savePicFunc(string _uri, string str) { //extract ajax? List <int> jsons = new List <int>(); string temp = str; int shift = 0; while (true) { var ind1 = temp.IndexOf("ajax.preload"); temp = temp.Substring(ind1 + 5); if (ind1 == -1) { break; } jsons.Add(ind1 + shift); shift += ind1 + 5; } List <string> jsonRefs = new List <string>(); foreach (var ind1 in jsons) { int brctcnt = 0; int brctcnt2 = 0; bool start = false; bool ajaxStart = false; StringBuilder sbb = new StringBuilder(); int? arrStart = null; for (int i2 = ind1; i2 < str.Length; i2++) { if (str[i2] == '(') { brctcnt2++; ajaxStart = true; } if (ajaxStart) { if (str[i2] == '[') { if (arrStart == null) { arrStart = i2; } brctcnt++; start = true; } } if (str[i2] == ']') { brctcnt--; } if (str[i2] == ')') { brctcnt2--; } if (start) { sbb.Append(str[i2]); } if (brctcnt == 0 && start) { break; } if (brctcnt2 == 0 && ajaxStart) { break; } } if (sbb.Length > 0 && arrStart != null) { var sub = str.Substring(arrStart.Value); byte[] byteArray = Encoding.ASCII.GetBytes(sub); MemoryStream stream = new MemoryStream(byteArray); var rdr = new StreamReader(stream); var rdr2 = new JsonTextReader(rdr); bool fetchNext = false; try { while (rdr2.Read()) { if (rdr2.ValueType == typeof(string)) { var str2 = (string)rdr2.Value; if (rdr2.TokenType == JsonToken.PropertyName) { if (str2 == "w_src" || str2 == "z_src") { fetchNext = true; } } else { if (fetchNext) { jsonRefs.Add(str2); fetchNext = false; } } } } } catch (JsonException ex) { } /*JArray o1 = JArray.Parse(sbb.ToString()); ; * * WalkNode(o1, n => * { * JToken token = n["w_src"]; * if (token != null && token.Type == JTokenType.String) * { * string wsrc = token.Value<string>(); * jsonRefs.Add(wsrc); * } * token = n["z_src"]; * if (token != null && token.Type == JTokenType.String) * { * string zsrc = token.Value<string>(); * jsonRefs.Add(zsrc); * } * });*/ } } var pi = new HttpPageInfo(); pi.Uri = _uri; Infos.Add(pi); //richTextBox2.Text = webBrowser1.DocumentText; var list = Stuff.ParseHtmlItems(str, "<meta", "/>"); var list2 = Stuff.ParseHtmlItems(str, "<a", "/a>"); var list3 = Stuff.ParseHtmlItems(str, "<img", ">"); int totalLinks = 0; var allLinks = jsonRefs.Union(list).Union(list2).Union(list3).ToArray(); foreach (var item in allLinks) { if (!item.Contains("http")) { continue; } var ar1 = item.Split(new char[] { ' ', '\"' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); var ww = ar1.Where(z => z.Contains("jpg") || z.Contains("png") || z.Contains("gif")); foreach (var _fr in ww) { var fr = _fr; try { var ind1 = fr.IndexOf("http"); if (ind1 > 0) { fr = fr.Substring(ind1); } string end = ".jpg"; if (fr.Contains("png")) { end = "png"; } if (fr.Contains("gif")) { end = "gif"; } var ind2 = fr.IndexOf(end); fr = fr.Substring(0, ind2 + end.Length); pi.Links.Add(new HttpPageInfo() { Uri = fr }); totalLinks++; listView2.Invoke((Action)(() => { listView2.Items.Add(new ListViewItem(fr) { Tag = fr }); })); links++; using (WebClient wc = new WebClient()) { var uri = new Uri(fr); if (strongNameSkip) { var nm = Path.Combine(saveTarget, uri.Segments.Last()); if (File.Exists(nm)) { skipped++; continue; } } var data = wc.DownloadData(fr); if (data.Length >= 0) { MemoryStream ms = new MemoryStream(data); if (showPics) { var img = Image.FromStream(ms); listView2.Invoke((Action)(() => { pictureBox1.Image = img; })); } ms.Seek(0, SeekOrigin.Begin); var hash = Stuff.CalcMD5(ms); if (md5cache.Add(hash)) { var fp = Path.Combine(saveTarget, uri.Segments.Last()); if (File.Exists(fp)) { throw new Exception("hash not found, but file exist"); } if (!useSizeFilter || ((data.Length / 1024) > sizeFilterKb)) { saved++; File.WriteAllBytes(fp, data); } else { skipped++; } /*File.WriteAllBytes(Path.Combine(textBox1.Text, DateTime.Now.Year + "-" + DateTime.Now.Month + "-" + DateTime.Now.Day + "-" + DateTime.Now.Minute + "-" + DateTime.Now.Hour + "-" + DateTime.Now.Second + ".png"), data);*/ //img.Save(Path.Combine(textBox1.Text, DateTime.Now.Year + "-" + DateTime.Now.Month + "-" + DateTime.Now.Day + "-" + DateTime.Now.Minute + "-" + DateTime.Now.Hour + "-" + DateTime.Now.Second + ".png")); } else { skipped++; } ms.Dispose(); } } break; } catch (Exception ex) { errors++; listView3.Invoke((Action)(() => { listView3.Items.Add(new ListViewItem(new string[] { fr, ex.Message }) { Tag = fr + ";" + ex }); })); } } } if (totalLinks == 0) { listView3.Invoke((Action)(() => { listView3.Items.Add(new ListViewItem(new string[] { "no links: " + _uri }) { Tag = _uri, BackColor = Color.Yellow }); })); } }