private static void ConvertPostLink(SiteUris uris, XmlReader r, SequentialTurtleWriter w, UnknownValueStore <string> unknownLinkTypeIds) { if (r.MoveToAttribute("LinkTypeId")) { switch (r.Value) { case "1": // linked if (r.MoveToAttribute("PostId")) { w.StartTriple(uris.CreatePostUri(r.Value)); if (r.MoveToAttribute("RelatedPostId")) { w.AddToTriple(uris.GeneralUris.LinkProperty, uris.CreatePostUri(r.Value)); } } break; case "3": // duplicate if (r.MoveToAttribute("RelatedPostId")) { w.StartTriple(uris.CreatePostUri(r.Value)); if (r.MoveToAttribute("PostId")) { w.AddToTriple(uris.GeneralUris.DuplicateProperty, uris.CreatePostUri(r.Value)); } } break; default: unknownLinkTypeIds.RegisterUnknownValue(r.Value); break; } } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No LinkTypeId attribute found on element {0}. Skipping element.", r.ReadOuterXml()); } }
private static void ConvertSiteList(GeneralUris generalUris, string tempDir, Uri baseUri, string destDir, VDS.RDF.INamespaceMapper nsMapper) { string srcFile = Path.Combine(tempDir, "Sites.xml"); ConsoleHelper.WriteMilestone("Downloading site list ..."); using (var client = new WebClient()) { client.DownloadFile(new Uri(baseUri, "Sites.xml"), srcFile); } Console.WriteLine(" done."); using (var destWriter = new SequentialTurtleWriter(File.CreateText(Path.Combine(destDir, "_sites.ttl")), nsMapper)) { using (var fs = File.OpenRead(srcFile)) { using (var reader = XmlReader.Create(fs)) { while (reader.NodeType != XmlNodeType.Element) { if (!reader.Read()) { ConsoleHelper.WriteErrorLine("No contents found in file {0}.", srcFile); return; } } if (reader.LocalName == "sitelist") { ConvertSites(generalUris, reader, destWriter); } else { ConsoleHelper.WriteWarningLine("Unknown root element \"{0}\". Skipping document.", reader.LocalName); } } } GlobalData.UpdateStats(destWriter); } Console.WriteLine("Conversion of site list completed."); }
private static void ConvertUser(SiteUris uris, XmlReader r, SequentialTurtleWriter w, ICollection <string> malformedIris, ref long totalMalformedIriCount) { Uri subjectUri; if (r.MoveToAttribute("Id")) { subjectUri = uris.CreateUserUri(r.Value); w.StartTriple(subjectUri); } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No Id attribute found on element {0}. Skipping element.", r.ReadOuterXml()); return; } w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.UserType); uris.LinkToSite(w); if (r.MoveToAttribute("DisplayName")) { w.AddToTriple(uris.GeneralUris.LabelProperty, r.Value); w.AddToTriple(uris.GeneralUris.UserNameProperty, r.Value); } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("Reputation")) { w.AddToTriple(uris.GeneralUris.ReputationProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("Location")) { w.AddToTriple(uris.GeneralUris.LocationProperty, r.Value); } if (r.MoveToAttribute("Age")) { w.AddToTriple(uris.GeneralUris.AgeProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("Views")) { w.AddToTriple(uris.GeneralUris.ViewCountProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("UpVotes")) { w.AddToTriple(uris.GeneralUris.UpVotesProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("DownVotes")) { w.AddToTriple(uris.GeneralUris.DownVotesProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("WebsiteUrl")) { string websiteUrl = r.Value; if (!string.IsNullOrWhiteSpace(websiteUrl)) { if (websiteUrl.ToLowerInvariant().StartsWith("http")) { websiteUrl = "http" + websiteUrl.Substring(4); } if (!emptyWebsiteRegex.IsMatch(websiteUrl.ToLowerInvariant())) { if (websiteRegex.IsMatch(websiteUrl)) { try { Uri homepageUrl = new Uri(websiteUrl); w.AddToTriple(uris.GeneralUris.WebsiteProperty, homepageUrl); } catch (UriFormatException) { totalMalformedIriCount++; if (malformedIris.Count < GlobalData.Options.MaxDisplayedMalformedIris) { malformedIris.Add(websiteUrl); } w.AddToTriple(uris.GeneralUris.WebsiteProperty, websiteUrl); } } else { totalMalformedIriCount++; if (malformedIris.Count < GlobalData.Options.MaxDisplayedMalformedIris) { malformedIris.Add(websiteUrl); } w.AddToTriple(uris.GeneralUris.WebsiteProperty, websiteUrl); } } } } if (r.MoveToAttribute("AccountId")) { w.AddToTriple(uris.GeneralUris.AccountProperty, uris.GeneralUris.CreateAccountUri(r.Value)); GlobalData.AccountIds.Add(r.Value); } if (r.MoveToAttribute("AboutMe")) { string desc = r.Value; if (!string.IsNullOrWhiteSpace(desc)) { w.AddToTriple(uris.GeneralUris.DescriptionProperty, desc); } } if (r.MoveToAttribute("LastAccessDate")) { w.AddToTriple(uris.GeneralUris.LastSeenProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } }
private static void ConvertPostHistoryItem(SiteUris uris, XmlReader r, SequentialTurtleWriter w, UnknownValueStore <string> unknownPostHistoryTypeIds) { Uri subjectUri; if (r.MoveToAttribute("Id")) { subjectUri = uris.CreatePostHistoryUri(r.Value); w.StartTriple(subjectUri); } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No Id attribute found on element {0}. Skipping element.", r.ReadOuterXml()); return; } if (r.MoveToAttribute("PostHistoryTypeId")) { switch (r.Value) { case "1": // initial title break; case "2": // initial body break; case "3": // initial tags break; case "4": // edit title break; case "5": // edit body break; case "6": // edit tags break; case "7": // rollback title break; case "8": // rollback body break; case "9": // rollback tags break; case "10": // post closed w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostClosureType); if (r.MoveToAttribute("Comment")) { switch (r.Value) { case "1": // Exact Duplicate case "101": // duplicate w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.DuplicateCloseReason); break; case "2": // Off-topic case "102": // Off-topic w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.OffTopicCloseReason); break; case "3": // Subjective and argumentative case "105": // Primarily opinion-based w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.SubjectiveCloseReason); break; case "4": // Not a real question w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.NotAQuestionCloseReason); break; case "7": // Too localized w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.TooLocalizedCloseReason); break; case "10": // General reference w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.GeneralReferenceCloseReason); break; case "20": // Noise or pointless w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.NoiseCloseReason); break; case "103": // Unclear what you're asking w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.UnclearCloseReason); break; case "104": // Too broad w.AddToTriple(uris.GeneralUris.CloseReasonProperty, uris.GeneralUris.TooBroadCloseReason); break; default: ConsoleHelper.WriteWarningLine("Unknown post close reason: {0}", r.Value); break; } } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "11": // post reopened w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostReopeningType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "12": // post deleted w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostDeletionType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "13": // post undeleted w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostUndeletionType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "14": // post locked w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostLockingType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "15": // post unlocked w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostUnlockingType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "16": // community owned break; case "17": // post migrated superseded with 35/36 break; case "18": // question merged break; case "19": // question protected w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostProtectionType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "20": // question unprotected w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.PostUnprotectionType); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } AddParticipants(uris, r, w); LinkToPost(uris, subjectUri, r, w); break; case "21": // post disassociated break; case "22": // question unmerged break; case "24": // suggested edit applied break; case "25": // post tweeted break; case "31": // comment discussion moved to chat break; case "33": // post notice added break; case "34": // post notice removed break; case "35": // post migrated away replaces 17 break; case "36": // post migrated here replaces 17 break; case "37": // post merge source break; case "38": // post merge destination break; default: unknownPostHistoryTypeIds.RegisterUnknownValue(r.Value); break; } } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No PostHistoryTypeId attribute found on element {0}. Skipping element.", r.ReadOuterXml()); } }
public static void Convert(GeneralUris generalUris, string srcFile, string destDir, SiteInfo website) { string fileNameOnly = Path.GetFileName(srcFile); Console.WriteLine("Processing {0} ...", fileNameOnly); var siteUris = new SiteUris(generalUris, website); var nsMapper = siteUris.CreateNamespaceMapper(); using (var destWriter = new SequentialTurtleWriter(File.CreateText(Path.Combine(destDir, website + "-" + Path.GetFileNameWithoutExtension(srcFile) + ".ttl")), nsMapper)) { using (var fs = File.OpenRead(srcFile)) { using (var reader = XmlReader.Create(fs)) { while (reader.NodeType != XmlNodeType.Element) { if (!reader.Read()) { ConsoleHelper.WriteErrorLine("No contents found in file {0}.", srcFile); return; } } switch (reader.LocalName) { case "badges": ConsoleHelper.WriteInfoLine("List of badges identified."); ConvertBadges(siteUris, reader, destWriter); break; case "comments": ConsoleHelper.WriteInfoLine("List of comments identified."); ConvertComments(siteUris, reader, destWriter); break; case "posthistory": ConsoleHelper.WriteInfoLine("List of posthistory identified."); ConvertPostHistory(siteUris, reader, destWriter); break; case "postlinks": ConsoleHelper.WriteInfoLine("List of postlinks identified."); ConvertPostLinks(siteUris, reader, destWriter); break; case "posts": ConsoleHelper.WriteInfoLine("List of posts identified."); ConvertPosts(siteUris, reader, destWriter); break; case "tags": ConsoleHelper.WriteInfoLine("List of tags identified."); ConvertTags(siteUris, reader, destWriter); break; case "users": ConsoleHelper.WriteInfoLine("List of users identified."); ConvertUsers(siteUris, reader, destWriter); break; case "votes": ConsoleHelper.WriteInfoLine("List of votes identified."); ConvertVotes(siteUris, reader, destWriter); break; default: ConsoleHelper.WriteWarningLine("Unknown root element \"{0}\". Skipping document.", reader.LocalName); break; } } } GlobalData.UpdateStats(destWriter); } Console.WriteLine("Conversion of {0} completed.", fileNameOnly); }
private static void ConvertVote(SiteUris uris, XmlReader r, SequentialTurtleWriter w, UnknownValueStore <string> unknownVoteTypeIds) { Uri subjectUri; if (r.MoveToAttribute("Id")) { subjectUri = uris.CreateVoteUri(r.Value); w.StartTriple(subjectUri); } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No Id attribute found on element {0}. Skipping element.", r.ReadOuterXml()); return; } if (r.MoveToAttribute("VoteTypeId")) { switch (r.Value) { case "1": // acceptance if (GlobalData.Options.FullTimeInfo) { w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.AcceptanceType); uris.LinkToSite(w); if (r.MoveToAttribute("PostId")) { w.StartTriple(uris.CreatePostUri(r.Value)); w.AddToTriple(uris.GeneralUris.AcceptanceProperty, subjectUri); w.StartTriple(subjectUri); } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } } break; case "2": // upvote if (GlobalData.Options.FullTimeInfo) { w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.UpVoteType); uris.LinkToSite(w); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("PostId")) { w.StartTriple(uris.CreatePostUri(r.Value)); w.AddToTriple(uris.GeneralUris.VoteProperty, subjectUri); w.StartTriple(subjectUri); } } break; case "3": // downvote if (GlobalData.Options.FullTimeInfo) { w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.DownVoteType); uris.LinkToSite(w); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("PostId")) { w.StartTriple(uris.CreatePostUri(r.Value)); w.AddToTriple(uris.GeneralUris.VoteProperty, subjectUri); w.StartTriple(subjectUri); } } break; case "4": // offensive break; case "5": // favorite if (GlobalData.Options.FullTimeInfo) { w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.FavoriteType); uris.LinkToSite(w); if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("PostId")) { w.AddToTriple(uris.GeneralUris.PostProperty, uris.CreatePostUri(r.Value)); } if (r.MoveToAttribute("UserId")) { w.StartTriple(uris.CreateUserUri(r.Value)); w.AddToTriple(uris.GeneralUris.FavoriteProperty, subjectUri); w.StartTriple(subjectUri); } } else { if (r.MoveToAttribute("UserId")) { w.StartTriple(uris.CreateUserUri(r.Value)); if (r.MoveToAttribute("PostId")) { w.AddToTriple(uris.GeneralUris.FavoriteProperty, uris.CreatePostUri(r.Value)); } w.StartTriple(subjectUri); } } break; case "6": // closed break; case "7": // reopened break; case "8": // bounty started w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.StartOfBountyType); if (r.MoveToAttribute("PostId")) { w.AddToTriple(uris.GeneralUris.PostProperty, uris.CreatePostUri(r.Value)); } if (r.MoveToAttribute("UserId")) { w.AddToTriple(uris.GeneralUris.DonorProperty, uris.CreateUserUri(r.Value)); } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("BountyAmount")) { w.AddToTriple(uris.GeneralUris.OfferedAmountProperty, long.Parse(r.Value)); } break; case "9": // bounty closed w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.EndOfBountyType); if (r.MoveToAttribute("PostId")) { w.AddToTriple(uris.GeneralUris.AnswerProperty, uris.CreatePostUri(r.Value)); } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("BountyAmount")) { w.AddToTriple(uris.GeneralUris.TransferredAmountProperty, long.Parse(r.Value)); } break; case "10": // deletion break; case "11": // undeletion break; case "12": // spam break; case "13": // moderator informed break; case "15": // under moderator review break; case "16": // approved edit suggestion break; default: unknownVoteTypeIds.RegisterUnknownValue(r.Value); break; } } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No VoteTypeId attribute found on element {0}. Skipping element.", r.ReadOuterXml()); } }
private static void ConvertPost(SiteUris uris, XmlReader r, SequentialTurtleWriter w, UnknownValueStore <string> unknownPostTypeIds) { Uri subjectUri; if (r.MoveToAttribute("Id")) { subjectUri = uris.CreatePostUri(r.Value); w.StartTriple(subjectUri); } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No Id attribute found on element {0}. Skipping element.", r.ReadOuterXml()); return; } if (r.MoveToAttribute("PostTypeId")) { switch (r.Value) { case "1": // question w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.QuestionType); uris.LinkToSite(w); if (r.MoveToAttribute("AcceptedAnswerId")) { w.AddToTriple(uris.GeneralUris.AcceptedAnswerProperty, uris.CreatePostUri(r.Value)); } if (r.MoveToAttribute("ViewCount")) { w.AddToTriple(uris.GeneralUris.ViewCountProperty, long.Parse(r.Value)); } if (r.MoveToAttribute("Title")) { w.AddToTriple(uris.GeneralUris.TitleProperty, r.Value); w.AddToTriple(uris.GeneralUris.LabelProperty, r.Value); } if (r.MoveToAttribute("Score")) { w.AddToTriple(uris.GeneralUris.ScoreProperty, long.Parse(r.Value)); } break; case "2": // answer w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.AnswerType); uris.LinkToSite(w); if (r.MoveToAttribute("ParentId")) { w.StartTriple(uris.CreatePostUri(r.Value)); w.AddToTriple(uris.GeneralUris.AnswerProperty, subjectUri); w.StartTriple(subjectUri); } else { ConsoleHelper.WriteWarningLine("Orphaned answer: {0}", subjectUri); } if (r.MoveToAttribute("Score")) { w.AddToTriple(uris.GeneralUris.ScoreProperty, long.Parse(r.Value)); } break; case "3": // orphaned tag wiki break; case "4": // tag info excerpt w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.TagExcerptType); break; case "5": // tag description w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.TagDescriptionType); break; case "6": // moderator nomination break; case "7": // "Wiki placeholder" (seems to only be the election description) //w.AddToTriple(uris.GeneralUris.TypeProperty, uris.GeneralUris.SiteInfoType); break; default: unknownPostTypeIds.RegisterUnknownValue(r.Value); break; } if (r.MoveToAttribute("CreationDate")) { w.AddToTriple(uris.GeneralUris.DateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("LastEditDate")) { w.AddToTriple(uris.GeneralUris.LastEditDateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("LastActivity")) { w.AddToTriple(uris.GeneralUris.LastActivityDateProperty, DateTime.Parse(r.Value, System.Globalization.CultureInfo.InvariantCulture)); } if (r.MoveToAttribute("OwnerUserId")) { w.AddToTriple(uris.GeneralUris.OwnerProperty, uris.CreateUserUri(r.Value)); } // TODO: LastEditorUserId (given in post history) // TODO: FavoriteCount (linked to users?) if (r.MoveToAttribute("Body")) { w.AddToTriple(uris.GeneralUris.DescriptionProperty, r.Value); } if (r.MoveToAttribute("Tags")) { w.AddToTriple(uris.GeneralUris.TagProperty, tagRegex.Matches(r.Value).Cast <Match>().Select(m => uris.CreateTagUri(m.Groups[1].Value))); } } else { r.MoveToElement(); ConsoleHelper.WriteErrorLine("No PostTypeId attribute found on element {0}. Skipping element.", r.ReadOuterXml()); } }
/// <summary> /// Downloads and converts the data. /// </summary> /// <param name="generalUris">An object that provides general URIs used in the exported dataset.</param> /// <param name="filelist">The URL of a filelist Xml file.</param> /// <exception cref="ArgumentNullException">Any of the arguments is <see langword="null"/>.</exception> private static void RetrieveData(GeneralUris generalUris, Uri filelist) { if (generalUris == null) { throw new ArgumentNullException("generalUris"); } if (filelist == null) { throw new ArgumentNullException("filelist"); } string tempDir = Path.Combine(BaseDir, "tmp"); string destDir = Path.Combine(BaseDir, "rdf"); Directory.CreateDirectory(tempDir); Directory.CreateDirectory(destDir); DateTime startTime = DateTime.Now; ConsoleHelper.WriteInfoLine("Current time: {0:yyyy-MM-dd HH:mm:ss}", startTime); if (!GlobalData.Options.OntologyOnly) { ConsoleHelper.WriteMilestone("Downloading files list ..."); using (var client = new WebClient()) { client.DownloadFile(filelist, Path.Combine(tempDir, "files.xml")); } Console.WriteLine(" done."); var files = LoadFilesList(Path.Combine(tempDir, "files.xml")).OrderBy(f => f).ToArray(); ConsoleHelper.WriteInfoLine("{0} file(s) in list, totalling to a compressed size of {1:F1} GB.", files.Length, (double)files.Sum(f => f.Size) / 1024 / 1024 / 1024); int processedFilesCount = 0; foreach (var f in files) { SiteInfo siteInfo; try { siteInfo = f.RetrieveSiteInfo(); } catch (ArgumentException ex) { ConsoleHelper.WriteErrorLine("Skipping file {0}, as it cannot be associated with a website.\n{1}", f, ex); continue; } if (IncludeSite(siteInfo)) { GlobalData.Sites[siteInfo.Id] = siteInfo; if (!GlobalData.Options.SiteListOnly) { string fn = f.Download(filelist, tempDir); string[] rawFiles = null; switch (Path.GetExtension(fn)) { case ".7z": rawFiles = ExtractSevenZipArchive(fn); break; default: ConsoleHelper.WriteWarningLine("File {0} has an unknown file extension.", fn); break; } if (rawFiles != null) { ConsoleHelper.WriteInfoLine("{0} file(s) extracted.", rawFiles.Length); foreach (var rawFile in rawFiles) { Converter.Convert(generalUris, rawFile, destDir, siteInfo); } } } processedFilesCount++; if (processedFilesCount >= GlobalData.Options.MaxFileCount) { break; } } } } GlobalInformationConverter.Convert(generalUris, tempDir, filelist, destDir); if (!GlobalData.Options.KeepTemporaryFiles) { Console.Write("Removing temporary files ... "); try { Directory.Delete(tempDir, true); Console.WriteLine(" done."); } catch { ConsoleHelper.WriteErrorLine("Please remove the directory {0} manually.", tempDir); } } Console.WriteLine(); GlobalData.PrintStats(); DateTime endTime = DateTime.Now; ConsoleHelper.WriteInfoLine("Current time: {0:yyyy-MM-dd HH:mm:ss}", endTime); ConsoleHelper.WriteInfoLine("Total duration: {0}", endTime - startTime); }