public flvStorewithDate(flvStore i) { this.filename = i.filename; this.line = i.line; this.linsrc = i.linsrc; this.flvName = i.flvName; if (rgdate.IsMatch(this.linsrc)) { Match m = rgdate.Match(this.linsrc); string syr = m.Groups["year"].Value; string smt = m.Groups["month"].Value; string sdy = m.Groups["day"].Value; DateTime d = new DateTime(int.Parse(syr), int.Parse(smt), int.Parse(sdy)); this.urlDate = d; } else { Console.WriteLine("Error"); } string[] spl = this.linsrc.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); if (spl.Length == 11) { this.compressedRecordSize = long.Parse(spl[9]); } else { throw new Exception("spl != 11"); } }
/// <summary> /// lets grab all the flv's we can find and store them in a directory /// </summary> /// <param name="miffix"></param> private static void part_2a(string miffix = "warccdx") { string pt = $@"{root}{miffix}\f\"; DirectoryInfo dinf = new DirectoryInfo(pt); List <flvStore> store = new List <flvStore>(); string pat0 = @".*?\/archives\/.*?\/(.*?\.flv).*"; Regex rgx0 = new Regex(pat0); FileInfo[] finfs = new DirectoryInfo(pt).GetFiles(); int cfile = 0; foreach (FileInfo f in finfs) { Console.WriteLine($"P2A In {cfile++} / {finfs.Length - 1}"); string[] a = File.ReadAllLines(f.FullName); int ln = 0; foreach (string x in a) { if (x.Contains(".flv")) { flvStore fl = new flvStore() { filename = f.Name, line = ln, linsrc = x }; if (rgx0.IsMatch(x)) { fl.flvName = rgx0.Match(x).Groups[1].Value; } store.Add(fl); } ln++; } } //dump to json. string jsonsrc = Newtonsoft.Json.JsonConvert.SerializeObject(store); File.WriteAllText($"{root}{miffix}.json", jsonsrc); }