private static ExtractorSettings GenerateConfigSettings( ) { var settings = new ExtractorSettings(); settings.AbsoluteSiteName = Prompt.GetString($"Fully qualified target website URL:", promptColor: ConsoleColor.DarkMagenta); settings.AllNodesFilename = Prompt.GetString($"\"All Nodes\" Xml Filename?", settings.AllNodesFilename, ConsoleColor.DarkMagenta); settings.AllRedirectsFilename = Prompt.GetString($"\"IIS Redirects\" Xml Filename:", settings.AllRedirectsFilename, ConsoleColor.DarkMagenta); settings.ConnectionString = Prompt.GetString("Connection String:", promptColor: ConsoleColor.DarkMagenta); settings.HashStringSalt = Prompt.GetString("Hash String Salt:", promptColor: ConsoleColor.DarkMagenta); settings.ObjectDirectory = Prompt.GetString($"Object (export) directory:", settings.ObjectDirectory, ConsoleColor.DarkMagenta); settings.RootNodeAliasPath = Prompt.GetString($"Root \"NodeAliasPath\":", settings.RootNodeAliasPath, ConsoleColor.DarkMagenta); var defaultOrderByColumns = string.Join(", ", settings.OrderByColumns); settings.OrderByColumns = Prompt.GetString($"OrderBy Columns (comma-delimited):", defaultOrderByColumns, ConsoleColor.DarkMagenta) .Split(',') .Select(s => s.Trim()) .ToArray(); var pageType = Prompt.GetString("PageType:", promptColor: ConsoleColor.DarkMagenta); settings.PageType = pageType; settings.PageTypeColumns = (Prompt.GetString("PageType Columns (comma-delimited):", promptColor: ConsoleColor.DarkMagenta) ?? string.Empty) .Split(',') .Select(s => s.Trim()) .ToArray(); settings.TopN = Prompt.GetInt($"Count:", settings.TopN, ConsoleColor.DarkMagenta); settings.XperienceUser = Prompt.GetString($"Xperience User name:", settings.XperienceUser, ConsoleColor.DarkMagenta); return(settings); }
public Extractor(ITokenizer tokenizer, IMorphAnalizer morphAnalizer, IChartParser parser, ExtractorSettings settings, ILogger <IExtractor> logger) { _tokenizer = tokenizer; _morphAnalizer = morphAnalizer; _parser = parser; _settings = settings; _logger = logger; }
public void AddParserSettings(ExtractorSettings parserSettings) { if (PriceTags == null) { PriceTags = new List <string>(); } if (UseParserSettings) { foreach (var priceHtmlPath in parserSettings.PriceHtmlPath) { PriceTags.Add(priceHtmlPath.Split('.').Last()); } PriceTags.Add(parserSettings.DiscountHtmlPath.Split('.').Last()); } }
private void PrepareObjectDirectory(ExtractorSettings settings) { if (Clean && Directory.Exists(settings.ObjectDirectory)) { Directory.Delete(settings.ObjectDirectory, true); } if (!Directory.Exists(settings.ObjectDirectory)) { Directory.CreateDirectory(settings.ObjectDirectory); } var pageTypePath = Path.Combine(settings.ObjectDirectory, settings.PageType); if (!Directory.Exists(pageTypePath)) { Directory.CreateDirectory(pageTypePath); } }
public static void Check(string text, string rules, IEnumerable <ExtractionDic> etalon, ExtractorSettings settings = null, IEnumerable <IExtension> extensions = null, IMorphAnalizer morph = null, params string[] rulesToExtract) { if (settings == null) { settings = new ExtractorSettings(); } var extractor = ExtractorFactory.Create(rules, settings, extensions: extensions, morph: morph); var result = extractor.Parse(text, rulesToExtract); _check(etalon.ToArray(), result.ToArray()); }
private void _SaveSfxStub(string exeToGenerate, SelfExtractorSaveOptions options) { string nameOfIconFile = null; string stubExe = null; string unpackedResourceDir = null; string tmpDir = null; try { if (File.Exists(exeToGenerate)) { if (Verbose) { StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate); } } if (!exeToGenerate.EndsWith(".exe")) { if (Verbose) { StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension."); } } // workitem 10553 tmpDir = TempFileFolder ?? Path.GetDirectoryName(exeToGenerate); stubExe = GenerateTempPathname(tmpDir, "exe"); // get the Ionic.Zip assembly Assembly a1 = typeof(ZipFile).Assembly; using (var csharp = new Microsoft.CSharp.CSharpCodeProvider (new Dictionary <string, string>() { { "CompilerVersion", "v2.0" } })) { // The following is a perfect opportunity for a linq query, but // I cannot use it. DotNetZip needs to run on .NET 2.0, // and using LINQ would break that. Here's what it would look // like: // // var settings = (from x in SettingsList // where x.Flavor == flavor // select x).First(); ExtractorSettings settings = null; foreach (var x in SettingsList) { if (x.Flavor == options.Flavor) { settings = x; break; } } // sanity check; should never happen if (settings == null) { throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", options.Flavor)); } // This is the list of referenced assemblies. Ionic.Zip is // needed here. Also if it is the winforms (gui) extractor, we // need other referenced assemblies, like // System.Windows.Forms.dll, etc. var cp = new System.CodeDom.Compiler.CompilerParameters(); cp.ReferencedAssemblies.Add(a1.Location); if (settings.ReferencedAssemblies != null) { foreach (string ra in settings.ReferencedAssemblies) { cp.ReferencedAssemblies.Add(ra); } } cp.GenerateInMemory = false; cp.GenerateExecutable = true; cp.IncludeDebugInformation = false; cp.CompilerOptions = ""; Assembly a2 = Assembly.GetExecutingAssembly(); // Use this to concatenate all the source code resources into a // single module. var sb = new System.Text.StringBuilder(); // In case there are compiler errors later, we allocate a source // file name now. If errors are detected, we'll spool the source // code as well as the errors (in comments) into that filename, // and throw an exception with the filename. Makes it easier to // diagnose. This should be rare; most errors happen only // during devlpmt of DotNetZip itself, but there are rare // occasions when they occur in other cases. string sourceFile = GenerateTempPathname(tmpDir, "cs"); // // debugging: enumerate the resources in this assembly // Console.WriteLine("Resources in this assembly:"); // foreach (string rsrc in a2.GetManifestResourceNames()) // { // Console.WriteLine(rsrc); // } // Console.WriteLine(); // all the source code is embedded in the DLL as a zip file. using (ZipFile zip = ZipFile.Read(a2.GetManifestResourceStream("Ionic.Zip.Resources.ZippedResources.zip"))) { // // debugging: enumerate the files in the embedded zip // Console.WriteLine("Entries in the embbedded zip:"); // foreach (ZipEntry entry in zip) // { // Console.WriteLine(entry.FileName); // } // Console.WriteLine(); unpackedResourceDir = GenerateTempPathname(tmpDir, "tmp"); if (String.IsNullOrEmpty(options.IconFile)) { // Use the ico file that is embedded into the Ionic.Zip // DLL itself. To do this we must unpack the icon to // the filesystem, in order to specify it on the cmdline // of csc.exe. This method will remove the unpacked // file later. System.IO.Directory.CreateDirectory(unpackedResourceDir); ZipEntry e = zip["zippedFile.ico"]; // Must not extract a readonly file - it will be impossible to // delete later. if ((e.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) { e.Attributes ^= FileAttributes.ReadOnly; } e.Extract(unpackedResourceDir); nameOfIconFile = Path.Combine(unpackedResourceDir, "zippedFile.ico"); cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile); } else { cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", options.IconFile); } cp.OutputAssembly = stubExe; if (options.Flavor == SelfExtractorFlavor.WinFormsApplication) { cp.CompilerOptions += " /target:winexe"; } if (!String.IsNullOrEmpty(options.AdditionalCompilerSwitches)) { cp.CompilerOptions += " " + options.AdditionalCompilerSwitches; } if (String.IsNullOrEmpty(cp.CompilerOptions)) { cp.CompilerOptions = null; } if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0)) { if (!Directory.Exists(unpackedResourceDir)) { System.IO.Directory.CreateDirectory(unpackedResourceDir); } foreach (string re in settings.CopyThroughResources) { string filename = Path.Combine(unpackedResourceDir, re); ExtractResourceToFile(a2, re, filename); // add the file into the target assembly as an embedded resource cp.EmbeddedResources.Add(filename); } } // add the Ionic.Utils.Zip DLL as an embedded resource cp.EmbeddedResources.Add(a1.Location); // file header sb.Append("// " + Path.GetFileName(sourceFile) + "\n") .Append("// --------------------------------------------\n//\n") .Append("// This SFX source file was generated by DotNetZip ") .Append(ZipFile.LibraryVersion.ToString()) .Append("\n// at ") .Append(System.DateTime.Now.ToString("yyyy MMMM dd HH:mm:ss")) .Append("\n//\n// --------------------------------------------\n\n\n"); // assembly attributes if (!String.IsNullOrEmpty(options.Description)) { sb.Append("[assembly: System.Reflection.AssemblyTitle(\"" + options.Description.Replace("\"", "") + "\")]\n"); } else { sb.Append("[assembly: System.Reflection.AssemblyTitle(\"DotNetZip SFX Archive\")]\n"); } if (!String.IsNullOrEmpty(options.ProductVersion)) { sb.Append("[assembly: System.Reflection.AssemblyInformationalVersion(\"" + options.ProductVersion.Replace("\"", "") + "\")]\n"); } // workitem string copyright = (String.IsNullOrEmpty(options.Copyright)) ? "Extractor: Copyright � Dino Chiesa 2008-2011" : options.Copyright.Replace("\"", ""); if (!String.IsNullOrEmpty(options.ProductName)) { sb.Append("[assembly: System.Reflection.AssemblyProduct(\"") .Append(options.ProductName.Replace("\"", "")) .Append("\")]\n"); } else { sb.Append("[assembly: System.Reflection.AssemblyProduct(\"DotNetZip\")]\n"); } sb.Append("[assembly: System.Reflection.AssemblyCopyright(\"" + copyright + "\")]\n") .Append(String.Format("[assembly: System.Reflection.AssemblyVersion(\"{0}\")]\n", ZipFile.LibraryVersion.ToString())); if (options.FileVersion != null) { sb.Append(String.Format("[assembly: System.Reflection.AssemblyFileVersion(\"{0}\")]\n", options.FileVersion.ToString())); } sb.Append("\n\n\n"); // Set the default extract location if it is available string extractLoc = options.DefaultExtractDirectory; if (extractLoc != null) { // remove double-quotes and replace slash with double-slash. // This, because the value is going to be embedded into a // cs file as a quoted string, and it needs to be escaped. extractLoc = extractLoc.Replace("\"", "").Replace("\\", "\\\\"); } string postExCmdLine = options.PostExtractCommandLine; if (postExCmdLine != null) { postExCmdLine = postExCmdLine.Replace("\\", "\\\\"); postExCmdLine = postExCmdLine.Replace("\"", "\\\""); } foreach (string rc in settings.ResourcesToCompile) { using (Stream s = zip[rc].OpenReader()) { if (s == null) { throw new ZipException(String.Format("missing resource '{0}'", rc)); } using (StreamReader sr = new StreamReader(s)) { while (sr.Peek() >= 0) { string line = sr.ReadLine(); if (extractLoc != null) { line = line.Replace("@@EXTRACTLOCATION", extractLoc); } line = line.Replace("@@REMOVE_AFTER_EXECUTE", options.RemoveUnpackedFilesAfterExecute.ToString()); line = line.Replace("@@QUIET", options.Quiet.ToString()); if (!String.IsNullOrEmpty(options.SfxExeWindowTitle)) { line = line.Replace("@@SFX_EXE_WINDOW_TITLE", options.SfxExeWindowTitle); } line = line.Replace("@@EXTRACT_EXISTING_FILE", ((int)options.ExtractExistingFile).ToString()); if (postExCmdLine != null) { line = line.Replace("@@POST_UNPACK_CMD_LINE", postExCmdLine); } sb.Append(line).Append("\n"); } } sb.Append("\n\n"); } } } string LiteralSource = sb.ToString(); #if DEBUGSFX // for debugging only string sourceModule = GenerateTempPathname(tmpDir, "cs"); using (StreamWriter sw = File.CreateText(sourceModule)) { sw.Write(LiteralSource); } Console.WriteLine("source: {0}", sourceModule); #endif var cr = csharp.CompileAssemblyFromSource(cp, LiteralSource); if (cr == null) { throw new SfxGenerationException("Cannot compile the extraction logic!"); } if (Verbose) { foreach (string output in cr.Output) { StatusMessageTextWriter.WriteLine(output); } } if (cr.Errors.Count != 0) { using (TextWriter tw = new StreamWriter(sourceFile)) { // first, the source we compiled tw.Write(LiteralSource); // now, append the compile errors tw.Write("\n\n\n// ------------------------------------------------------------------\n"); tw.Write("// Errors during compilation: \n//\n"); string p = Path.GetFileName(sourceFile); foreach (System.CodeDom.Compiler.CompilerError error in cr.Errors) { tw.Write(String.Format("// {0}({1},{2}): {3} {4}: {5}\n//\n", p, // 0 error.Line, // 1 error.Column, // 2 error.IsWarning ? "Warning" : "error", // 3 error.ErrorNumber, // 4 error.ErrorText)); // 5 } } throw new SfxGenerationException(String.Format("Errors compiling the extraction logic! {0}", sourceFile)); } OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor); // Now, copy the resulting EXE image to the _writestream. // Because this stub exe is being saved first, the effect will be to // concatenate the exe and the zip data together. using (System.IO.Stream input = System.IO.File.OpenRead(stubExe)) { byte[] buffer = new byte[4000]; int n = 1; while (n != 0) { n = input.Read(buffer, 0, buffer.Length); if (n != 0) { WriteStream.Write(buffer, 0, n); } } } } OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive); } finally { try { if (Directory.Exists(unpackedResourceDir)) { try { Directory.Delete(unpackedResourceDir, true); } catch (System.IO.IOException exc1) { StatusMessageTextWriter.WriteLine("Warning: Exception: {0}", exc1); } } if (File.Exists(stubExe)) { try { File.Delete(stubExe); } catch (System.IO.IOException exc1) { StatusMessageTextWriter.WriteLine("Warning: Exception: {0}", exc1); } } } catch (System.IO.IOException) { } } return; }
/// <summary> /// Saves the ZipFile instance to a self-extracting zip archive. /// </summary> /// /// <remarks> /// /// <para> /// The generated exe image will execute on any machine that has the .NET Framework 2.0 /// installed on it. /// </para> /// /// <para> /// There are two "flavors" of self-extracting archive. The <c>WinFormsApplication</c> /// version will pop up a GUI and allow the user to select a target directory into which /// to extract. There's also a checkbox allowing the user to specify to overwrite /// existing files, and another checkbox to allow the user to request that Explorer be /// opened to see the extracted files after extraction. The other flavor is /// <c>ConsoleApplication</c>. A self-extractor generated with that flavor setting will /// run from the command line. It accepts command-line options to set the overwrite /// behavior, and to specify the target extraction directory. /// </para> /// /// <para> /// There are a few temporary files created during the saving to a self-extracting zip. /// These files are created in the directory pointed to by /// <see cref="ZipFile.TempFileFolder"/>, which defaults to <see cref="System.IO.Path.GetTempPath"/>. /// These temporary files are removed upon successful completion of this method. /// </para> /// /// <para> /// When a user runs the SFX, the user's personal directory /// (<see cref="Environment.SpecialFolder.Personal"/>) /// will be used as the default extract location. /// The user who runs the SFX will have the opportunity to change the extract /// directory before extracting. /// </para> /// /// <para> /// NB: This method is not available in the version of DotNetZip /// build for the .NET Compact Framework, nor in the "Reduced" DotNEtZip library. /// </para> /// /// </remarks> /// /// <example> /// <code> /// string DirectoryPath = "c:\\Documents\\Project7"; /// using (ZipFile zip = new ZipFile()) /// { /// zip.AddDirectory(DirectoryPath, System.IO.Path.GetFileName(DirectoryPath)); /// zip.Comment = "This will be embedded into a self-extracting console-based exe"; /// zip.SaveSelfExtractor("archive.exe", SelfExtractorFlavor.ConsoleApplication); /// } /// </code> /// <code lang="VB"> /// Dim DirectoryPath As String = "c:\Documents\Project7" /// Using zip As New ZipFile() /// zip.AddDirectory(DirectoryPath, System.IO.Path.GetFileName(DirectoryPath)) /// zip.Comment = "This will be embedded into a self-extracting console-based exe" /// zip.SaveSelfExtractor("archive.exe", SelfExtractorFlavor.ConsoleApplication) /// End Using /// </code> /// </example> /// /// <param name="exeToGenerate">a pathname, possibly fully qualified, to be created. Typically it will end in an .exe extension.</param> /// <param name="flavor">Indicates whether a Winforms or Console self-extractor is desired.</param> public void SaveSelfExtractor(string exeToGenerate, SelfExtractorFlavor flavor) { if (File.Exists(exeToGenerate)) { if (Verbose) { StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate); } } if (!exeToGenerate.EndsWith(".exe")) { if (Verbose) { StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension."); } } string TempZipFile = SfxSaveTemporary(); OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive); if (TempZipFile == null) { return; // cancelled } // look for myself (ZipFile will be present in the Ionic.Utils.Zip assembly) Assembly a1 = typeof(ZipFile).Assembly; //Console.WriteLine("DotNetZip assembly loc: {0}", a1.Location); Microsoft.CSharp.CSharpCodeProvider csharp = new Microsoft.CSharp.CSharpCodeProvider(); // Perfect opportunity for a linq query, but I cannot use it. // The DotNetZip library can compile into 2.0, but needs to run on .NET 2.0. // Using LINQ would break that. Here's what it would look like: // // var settings = (from x in SettingsList // where x.Flavor == flavor // select x).First(); ExtractorSettings settings = null; foreach (var x in SettingsList) { if (x.Flavor == flavor) { settings = x; break; } } if (settings == null) { throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", flavor)); } // This is the list of referenced assemblies. Ionic.Utils.Zip is needed here. // Also if it is the winforms (gui) extractor, we need other referenced assemblies. System.CodeDom.Compiler.CompilerParameters cp = new System.CodeDom.Compiler.CompilerParameters(); cp.ReferencedAssemblies.Add(a1.Location); if (settings.ReferencedAssemblies != null) { foreach (string ra in settings.ReferencedAssemblies) { cp.ReferencedAssemblies.Add(ra); } } cp.GenerateInMemory = false; cp.GenerateExecutable = true; cp.IncludeDebugInformation = false; cp.OutputAssembly = exeToGenerate; Assembly a2 = Assembly.GetExecutingAssembly(); string TempDir = GenerateUniquePathname("tmp", null); if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0)) { System.IO.Directory.CreateDirectory(TempDir); int n = 0; byte[] bytes = new byte[1024]; foreach (string re in settings.CopyThroughResources) { string filename = Path.Combine(TempDir, re); using (Stream instream = a2.GetManifestResourceStream(re)) { using (FileStream outstream = File.OpenWrite(filename)) { do { n = instream.Read(bytes, 0, bytes.Length); outstream.Write(bytes, 0, n); } while (n > 0); } } // add the embedded resource in our own assembly into the target assembly as an embedded resource cp.EmbeddedResources.Add(filename); } } // add the zip file as an embedded resource cp.EmbeddedResources.Add(TempZipFile); // add the Ionic.Utils.Zip DLL as an embedded resource cp.EmbeddedResources.Add(a1.Location); //Console.WriteLine("Resources in this assembly:"); //foreach (string rsrc in a2.GetManifestResourceNames()) //{ // Console.WriteLine(rsrc); //} //Console.WriteLine(); //Console.WriteLine("reading source code resources:"); // concatenate all the source code resources into a single module var sb = new System.Text.StringBuilder(); // set the default extract location if it is available bool wantCodeReplace = (flavor == SelfExtractorFlavor.WinFormsApplication && _defaultExtractLocation != null); if (wantCodeReplace) { _defaultExtractLocation = _defaultExtractLocation.Replace("\"", ""); } foreach (string rc in settings.ResourcesToCompile) { //Console.WriteLine(" trying to read stream: ({0})", rc); Stream s = a2.GetManifestResourceStream(rc); using (StreamReader sr = new StreamReader(s)) { while (sr.Peek() >= 0) { string line = sr.ReadLine(); if (wantCodeReplace) { line = line.Replace("@@VALUE", _defaultExtractLocation); } sb.Append(line).Append("\n"); } } sb.Append("\n\n"); } string LiteralSource = sb.ToString(); System.CodeDom.Compiler.CompilerResults cr = csharp.CompileAssemblyFromSource(cp, LiteralSource); if (cr == null) { throw new SfxGenerationException("Cannot compile the extraction logic!"); } if (Verbose) { foreach (string output in cr.Output) { StatusMessageTextWriter.WriteLine(output); } } if (cr.Errors.Count != 0) { throw new SfxGenerationException("Errors compiling the extraction logic!"); } OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor); try { if (Directory.Exists(TempDir)) { try { Directory.Delete(TempDir, true); } catch { } } if (File.Exists(TempZipFile)) { try { File.Delete(TempZipFile); } catch { } } } catch { } OnSaveCompleted(); if (Verbose) { StatusMessageTextWriter.WriteLine("Created self-extracting zip file {0}.", cr.PathToAssembly); } return; // catch (Exception e1) // { // StatusMessageTextWriter.WriteLine("****Exception: " + e1); // throw; // } // return; }
private void Extract(IConsole console, ExtractorSettings settings) { var redirects = new List <(string, string)>(); // Create the output directories if needed if (!DryRun) { PrepareObjectDirectory(settings); } //Doc to save the actual content as xml var xmlDocument = new XmlDocument(); var parentElem = xmlDocument.CreateNode("element", settings.PageType.Replace(".", "-") + "s", ""); //Fire up our Kentico Xperience instance ConnectionHelper.ConnectionString = settings.ConnectionString; ValidationHelper.HashStringSalt = settings.HashStringSalt; CMSApplication.Init(); // Gets an object representing a specific Kentico user UserInfo user = UserInfoProvider.GetUserInfo(settings.XperienceUser); // Sets the context of the user using (new CMSActionContext(user)) { var wc = new WebClient(); //Get me all published pages of the specified page type on the site var posts = DocumentHelper.GetDocuments(settings.PageType) .Columns(settings.PageTypeColumns) .Path(settings.RootNodeAliasPath, PathTypeEnum.Children) .PublishedVersion() .Published() .TopN(settings.TopN) .OrderBy(settings.OrderByColumns); console.WriteLine($"{posts.Count} nodes found."); console.WriteLine("Exporting node data..."); int nodeOrder = 1; foreach (TreeNode post in posts) { //Create a new AgilityPack doc for working with the page HtmlDocument doc = new HtmlDocument(); doc.OptionOutputAsXml = false; //if true adds a weird span tag doc.DocumentNode.AppendChild(HtmlNode.CreateNode("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")); doc.DocumentNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine)); string nodeAliasPath = post.NodeAliasPath; nodeAliasPath = nodeAliasPath.Split('/').Last(); //clean it up to just the slug // Add redirect to the list redirects.Add((post.NodeAliasPath, nodeAliasPath)); // TODO: use FormInfo to determine "DocumentName" field string nodeTitle = post.DocumentName; //Create a parent folder path in the data directory to store all xml about this node in a structured way on the file system string nodeDirPath = Path.Combine(settings.ObjectDirectory, settings.PageType, nodeAliasPath.Replace("/", "-")); if (!Directory.Exists(nodeDirPath)) { Directory.CreateDirectory(nodeDirPath); } //Create root xml Element of the pageTypeName ex. <cms-blogpost> // and start saving fields as attributes and child elements in this xml structure var rootNode = HtmlNode.CreateNode($"<{settings.PageType.Replace( ".", "-" )} />"); rootNode.Attributes.Append("originalAliasPath"); rootNode.SetAttributeValue("originalAliasPath", post.NodeAliasPath); rootNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine)); var aliasNode = HtmlNode.CreateNode($"<newaliaspath/>"); aliasNode.InnerHtml = nodeAliasPath; rootNode.AppendChild(aliasNode); var orderNode = HtmlNode.CreateNode($"<newnodeorder/>"); orderNode.InnerHtml = nodeOrder.ToString(); rootNode.AppendChild(orderNode); nodeOrder++; //Kentico dynamic properties or "With the Coupled Columns of the page type" //Iterate through all of the fields of a document in order to serialize them to xml foreach (var dprop in post.Properties) { if (Array.IndexOf(settings.PageTypeColumns, dprop) > 0) { //Create xml Element that represents each field ex. <blogpostsummary>data</blogpostsummary> var node = HtmlNode.CreateNode($"<{dprop} />"); var data = post.GetValue(dprop, ""); //Special stuff for Mcbeev Blog Post Summary (can be ignored for others) if (dprop.ToLower().Contains("summary")) { //Remove wrapped p tag on Mcbeev Blog Post Summary HtmlDocument docSummary = new HtmlDocument(); docSummary.LoadHtml(data); var newSummaryNode = HtmlNode.CreateNode($"<newblogpostsummary/>"); try { if (docSummary.DocumentNode.SelectSingleNode("//p") != null) { newSummaryNode.InnerHtml = docSummary.DocumentNode.SelectSingleNode("//p").InnerText; } else { newSummaryNode.InnerHtml = docSummary.DocumentNode.InnerText; } } catch (Exception ex) { //todo console.WriteLine(ex.Message); } rootNode.AppendChild(newSummaryNode); rootNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine)); } //This is WYSIWYG content field most likely so try to resolve things as Kentico would if (dprop.ToLower().Contains("title") || dprop.ToLower().Contains("body") || dprop.ToLower().Contains("content") || dprop.ToLower().Contains("summary")) { //Macro resolve inside of the content (handle K# macros in the text) data = MacroResolver.Resolve(data); //Url resolve inside of the content (handle relative urls into absolute urls) important! data = HTMLHelper.ResolveUrls(data, settings.AbsoluteSiteName, true); //Widget identifier (try to save a list of all Portal Inline Widgets that no longer work in MVC) if (data.Contains("CMSInlineControl")) { console.WriteLine("Found a widget in this content."); // TODO Make this a real log // //Render the value as html to HP NodedataRootDirectoryName // HtmlDocument docWidget = new HtmlDocument(); // docWidget.LoadHtml(data); // var widgets = docWidget.DocumentNode.SelectNodes("//object[contains(@type, 'Widget')]"); // foreach (var w in widgets) // { // console.WriteLine(w.OuterHtml); // } } //Media library identifier if (data.Contains("<img ")) { HtmlDocument docImg = new HtmlDocument(); docImg.LoadHtml(data); var ImageURLs = docImg.DocumentNode.Descendants("img") .Select(e => e.GetAttributeValue("src", null)) .Where(s => !String.IsNullOrEmpty(s)); foreach (var oldImageURL in ImageURLs) { //Skip mp3s for now if (oldImageURL.EndsWith(".mp3") || oldImageURL.Contains(".mp3?")) { continue; } console.WriteLine($"trying to save {oldImageURL} ..."); var safeName = oldImageURL.Split(new char[] { '?' })[0]; safeName = safeName.Replace(".aspx", ""); //Save the images locally if we can download from the "real" running site try { wc.DownloadFile(oldImageURL, Path.Combine(nodeDirPath, Path.GetFileName(safeName))); } catch (Exception ex) { console.WriteLine(ex.Message); } //rewrite the image path back in the data node // TODO - make this a config parameter and not hard coded to Mcbeev.com data = data.Replace(oldImageURL, "/MBV/media/blog/" + nodeAliasPath.Replace("/", "-") + "/" + oldImageURL.Split(new char[] { '/' }).Last()); //replace the odd .png.aspx thing data = data.Replace(".png.aspx", ".png"); } } //Use HTMLAgility Pack to fix non closed image tags HtmlNode.ElementsFlags["img"] = HtmlElementFlag.Closed; HtmlDocument docContent = new HtmlDocument(); docContent.LoadHtml(data); data = $"<![CDATA[{docContent.DocumentNode.OuterHtml}]]>"; } //Most likely image only field and not a WYSIWYG field // TODO - is there a better way to check for this? if ((data.Length > 2) && (data.StartsWith("~/"))) { data = data.Replace("~/", settings.AbsoluteSiteName); console.WriteLine($"trying to save {data} ..."); var safeName = data.Split(new char[] { '?' })[0]; safeName = safeName.Replace(".aspx", ""); wc.DownloadFile(data, Path.Combine(nodeDirPath, Path.GetFileName(safeName))); //rewrite the image path back in the data node safeName = "/MBV/media/blog/" + nodeAliasPath.Replace("/", "-") + "/" + safeName.Split(new char[] { '/' }).Last(); data = $"<![CDATA[{safeName}]]>"; } node.AppendChild(doc.CreateTextNode(data)); rootNode.AppendChild(node); rootNode.AppendChild(HtmlNode.CreateNode("\r\n")); } } doc.DocumentNode.AppendChild(rootNode); //Save the single node to its own XML document in its directory doc.Save($@"{nodeDirPath}\{nodeAliasPath.Replace( "/", "-" )}.xml"); //Chop off the header to prepare for bundling doc.DocumentNode.ChildNodes[0].Remove(); //add this node to all nodes in main XML doc parentElem.InnerXml += doc.DocumentNode.OuterHtml; xmlDocument.AppendChild(parentElem); } if (!DryRun) { WriteRedirects(redirects, settings); xmlDocument.Save(Path.Combine(settings.ObjectDirectory, settings.PageType, settings.AllNodesFilename)); console.WriteLine($"Extraction to {settings.ObjectDirectory} succeeded"); } else { console.WriteLine("Dry run succeeded"); } } }
private void _SaveSfxStub(string exeToGenerate, SelfExtractorFlavor flavor, string defaultExtractLocation, string postExtractCmdLine, string nameOfIconFile) { bool removeIconFile = false; string StubExe = null; string TempDir = null; try { if (File.Exists(exeToGenerate)) { if (Verbose) { StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate); } } if (!exeToGenerate.EndsWith(".exe")) { if (Verbose) { StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension."); } } StubExe = GenerateTempPathname("exe", null); // get the Ionic.Zip assembly Assembly a1 = typeof(ZipFile).Assembly; Microsoft.CSharp.CSharpCodeProvider csharp = new Microsoft.CSharp.CSharpCodeProvider(); // Perfect opportunity for a linq query, but I cannot use it. // The DotNetZip library can compile into 2.0, but needs to run on .NET 2.0. // Using LINQ would break that. Here's what it would look like: // // var settings = (from x in SettingsList // where x.Flavor == flavor // select x).First(); ExtractorSettings settings = null; foreach (var x in SettingsList) { if (x.Flavor == flavor) { settings = x; break; } } if (settings == null) { throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", flavor)); } // This is the list of referenced assemblies. Ionic.Zip is needed here. // Also if it is the winforms (gui) extractor, we need other referenced assemblies, // like System.Windows.Forms.dll, etc. System.CodeDom.Compiler.CompilerParameters cp = new System.CodeDom.Compiler.CompilerParameters(); cp.ReferencedAssemblies.Add(a1.Location); if (settings.ReferencedAssemblies != null) { foreach (string ra in settings.ReferencedAssemblies) { cp.ReferencedAssemblies.Add(ra); } } cp.GenerateInMemory = false; cp.GenerateExecutable = true; cp.IncludeDebugInformation = false; cp.CompilerOptions = ""; Assembly a2 = Assembly.GetExecutingAssembly(); if (nameOfIconFile == null) { removeIconFile = true; nameOfIconFile = GenerateTempPathname("ico", null); ExtractResourceToFile(a2, "Ionic.Zip.Resources.zippedFile.ico", nameOfIconFile); cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile); } else if (nameOfIconFile != "") { cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile); } //cp.IncludeDebugInformation = true; cp.OutputAssembly = StubExe; if (flavor == SelfExtractorFlavor.WinFormsApplication) { cp.CompilerOptions += " /target:winexe"; } if (cp.CompilerOptions == "") { cp.CompilerOptions = null; } TempDir = GenerateTempPathname("tmp", null); if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0)) { System.IO.Directory.CreateDirectory(TempDir); foreach (string re in settings.CopyThroughResources) { string filename = Path.Combine(TempDir, re); ExtractResourceToFile(a2, re, filename); // add the file into the target assembly as an embedded resource cp.EmbeddedResources.Add(filename); } } // add the Ionic.Utils.Zip DLL as an embedded resource cp.EmbeddedResources.Add(a1.Location); //Console.WriteLine("Resources in this assembly:"); //foreach (string rsrc in a2.GetManifestResourceNames()) //{ // Console.WriteLine(rsrc); //} //Console.WriteLine(); //Console.WriteLine("reading source code resources:"); // concatenate all the source code resources into a single module var sb = new System.Text.StringBuilder(); // assembly attributes sb.Append("[assembly: System.Reflection.AssemblyTitle(\"DotNetZip SFX Archive\")]\n"); sb.Append("[assembly: System.Reflection.AssemblyProduct(\"ZipLibrary\")]\n"); sb.Append("[assembly: System.Reflection.AssemblyCopyright(\"Copyright © Dino Chiesa 2008, 2009\")]\n"); sb.Append(String.Format("[assembly: System.Reflection.AssemblyVersion(\"{0}\")]\n\n", ZipFile.LibraryVersion.ToString())); // Set the default extract location if it is available, and if supported. bool haveLocation = (defaultExtractLocation != null); if (haveLocation) { defaultExtractLocation = defaultExtractLocation.Replace("\"", "").Replace("\\", "\\\\"); } foreach (string rc in settings.ResourcesToCompile) { //Console.WriteLine(" trying to read stream: ({0})", rc); Stream s = a2.GetManifestResourceStream(rc); if (s == null) { throw new ZipException(String.Format("missing resource '{0}'", rc)); } using (StreamReader sr = new StreamReader(s)) { while (sr.Peek() >= 0) { string line = sr.ReadLine(); if (haveLocation) { line = line.Replace("@@EXTRACTLOCATION", defaultExtractLocation); } if (postExtractCmdLine != null) { line = line.Replace("@@POST_UNPACK_CMD_LINE", postExtractCmdLine.Replace("\\", "\\\\")); } sb.Append(line).Append("\n"); } } sb.Append("\n\n"); } string LiteralSource = sb.ToString(); #if DEBUGSFX // for debugging only string sourceModule = GenerateTempPathname("cs", null); using (StreamWriter sw = File.CreateText(sourceModule)) { sw.Write(LiteralSource); } Console.WriteLine("source: {0}", sourceModule); #endif System.CodeDom.Compiler.CompilerResults cr = csharp.CompileAssemblyFromSource(cp, LiteralSource); if (cr == null) { throw new SfxGenerationException("Cannot compile the extraction logic!"); } if (Verbose) { foreach (string output in cr.Output) { StatusMessageTextWriter.WriteLine(output); } } if (cr.Errors.Count != 0) { //Console.ReadLine(); string sourcefile = GenerateTempPathname("cs", null); using (TextWriter tw = new StreamWriter(sourcefile)) { tw.Write(LiteralSource); } throw new SfxGenerationException(String.Format("Errors compiling the extraction logic! {0}", sourcefile)); } OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor); // Now, copy the resulting EXE image to the _writestream. // Because this stub exe is being saved first, the effect will be to // concatenate the exe and the zip data together. using (System.IO.Stream input = System.IO.File.OpenRead(StubExe)) { byte[] buffer = new byte[4000]; int n = 1; while (n != 0) { n = input.Read(buffer, 0, buffer.Length); if (n != 0) { WriteStream.Write(buffer, 0, n); } } } OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive); } finally { try { if (Directory.Exists(TempDir)) { try { Directory.Delete(TempDir, true); } catch { } } if (File.Exists(StubExe)) { try { File.Delete(StubExe); } catch { } } if (removeIconFile && File.Exists(nameOfIconFile)) { try { File.Delete(nameOfIconFile); } catch { } } } catch { } } return; }
private async Task WriteConfigAsync(ExtractorSettings settings) { using var file = new FileStream(Output, FileMode.Create); await JsonSerializer.SerializeAsync(file, settings, new JsonSerializerOptions { WriteIndented = true }); }
public static void Validate(this ExtractorSettings settings) { var context = new ValidationContext(settings, serviceProvider: null, items: null); Validator.ValidateObject(settings, context, true); }
public StartTerminalsCreator(ExtractorSettings settings) { _settings = settings; }