private static ExtractorSettings GenerateConfigSettings( )
{
var settings = new ExtractorSettings();
settings.AbsoluteSiteName = Prompt.GetString($"Fully qualified target website URL:", promptColor: ConsoleColor.DarkMagenta);
settings.AllNodesFilename = Prompt.GetString($"\"All Nodes\" Xml Filename?", settings.AllNodesFilename, ConsoleColor.DarkMagenta);
settings.AllRedirectsFilename = Prompt.GetString($"\"IIS Redirects\" Xml Filename:", settings.AllRedirectsFilename, ConsoleColor.DarkMagenta);
settings.ConnectionString = Prompt.GetString("Connection String:", promptColor: ConsoleColor.DarkMagenta);
settings.HashStringSalt = Prompt.GetString("Hash String Salt:", promptColor: ConsoleColor.DarkMagenta);
settings.ObjectDirectory = Prompt.GetString($"Object (export) directory:", settings.ObjectDirectory, ConsoleColor.DarkMagenta);
settings.RootNodeAliasPath = Prompt.GetString($"Root \"NodeAliasPath\":", settings.RootNodeAliasPath, ConsoleColor.DarkMagenta);
var defaultOrderByColumns = string.Join(", ", settings.OrderByColumns);
settings.OrderByColumns = Prompt.GetString($"OrderBy Columns (comma-delimited):", defaultOrderByColumns, ConsoleColor.DarkMagenta)
.Split(',')
.Select(s => s.Trim())
.ToArray();
var pageType = Prompt.GetString("PageType:", promptColor: ConsoleColor.DarkMagenta);
settings.PageType = pageType;
settings.PageTypeColumns = (Prompt.GetString("PageType Columns (comma-delimited):", promptColor: ConsoleColor.DarkMagenta) ?? string.Empty)
.Split(',')
.Select(s => s.Trim())
.ToArray();
settings.TopN = Prompt.GetInt($"Count:", settings.TopN, ConsoleColor.DarkMagenta);
settings.XperienceUser = Prompt.GetString($"Xperience User name:", settings.XperienceUser, ConsoleColor.DarkMagenta);
return(settings);
}
public Extractor(ITokenizer tokenizer,
IMorphAnalizer morphAnalizer,
IChartParser parser,
ExtractorSettings settings,
ILogger <IExtractor> logger)
{
_tokenizer = tokenizer;
_morphAnalizer = morphAnalizer;
_parser = parser;
_settings = settings;
_logger = logger;
}
public void AddParserSettings(ExtractorSettings parserSettings)
{
if (PriceTags == null)
{
PriceTags = new List <string>();
}
if (UseParserSettings)
{
foreach (var priceHtmlPath in parserSettings.PriceHtmlPath)
{
PriceTags.Add(priceHtmlPath.Split('.').Last());
}
PriceTags.Add(parserSettings.DiscountHtmlPath.Split('.').Last());
}
}
private void PrepareObjectDirectory(ExtractorSettings settings)
{
if (Clean && Directory.Exists(settings.ObjectDirectory))
{
Directory.Delete(settings.ObjectDirectory, true);
}
if (!Directory.Exists(settings.ObjectDirectory))
{
Directory.CreateDirectory(settings.ObjectDirectory);
}
var pageTypePath = Path.Combine(settings.ObjectDirectory, settings.PageType);
if (!Directory.Exists(pageTypePath))
{
Directory.CreateDirectory(pageTypePath);
}
}
public static void Check(string text,
string rules,
IEnumerable <ExtractionDic> etalon,
ExtractorSettings settings = null,
IEnumerable <IExtension> extensions = null,
IMorphAnalizer morph = null,
params string[] rulesToExtract)
{
if (settings == null)
{
settings = new ExtractorSettings();
}
var extractor = ExtractorFactory.Create(rules,
settings,
extensions: extensions,
morph: morph);
var result = extractor.Parse(text, rulesToExtract);
_check(etalon.ToArray(), result.ToArray());
}
private void _SaveSfxStub(string exeToGenerate, SelfExtractorSaveOptions options)
{
string nameOfIconFile = null;
string stubExe = null;
string unpackedResourceDir = null;
string tmpDir = null;
try
{
if (File.Exists(exeToGenerate))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate);
}
}
if (!exeToGenerate.EndsWith(".exe"))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension.");
}
}
// workitem 10553
tmpDir = TempFileFolder ?? Path.GetDirectoryName(exeToGenerate);
stubExe = GenerateTempPathname(tmpDir, "exe");
// get the Ionic.Zip assembly
Assembly a1 = typeof(ZipFile).Assembly;
using (var csharp = new Microsoft.CSharp.CSharpCodeProvider
(new Dictionary <string, string>()
{
{ "CompilerVersion", "v2.0" }
})) {
// The following is a perfect opportunity for a linq query, but
// I cannot use it. DotNetZip needs to run on .NET 2.0,
// and using LINQ would break that. Here's what it would look
// like:
//
// var settings = (from x in SettingsList
// where x.Flavor == flavor
// select x).First();
ExtractorSettings settings = null;
foreach (var x in SettingsList)
{
if (x.Flavor == options.Flavor)
{
settings = x;
break;
}
}
// sanity check; should never happen
if (settings == null)
{
throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", options.Flavor));
}
// This is the list of referenced assemblies. Ionic.Zip is
// needed here. Also if it is the winforms (gui) extractor, we
// need other referenced assemblies, like
// System.Windows.Forms.dll, etc.
var cp = new System.CodeDom.Compiler.CompilerParameters();
cp.ReferencedAssemblies.Add(a1.Location);
if (settings.ReferencedAssemblies != null)
{
foreach (string ra in settings.ReferencedAssemblies)
{
cp.ReferencedAssemblies.Add(ra);
}
}
cp.GenerateInMemory = false;
cp.GenerateExecutable = true;
cp.IncludeDebugInformation = false;
cp.CompilerOptions = "";
Assembly a2 = Assembly.GetExecutingAssembly();
// Use this to concatenate all the source code resources into a
// single module.
var sb = new System.Text.StringBuilder();
// In case there are compiler errors later, we allocate a source
// file name now. If errors are detected, we'll spool the source
// code as well as the errors (in comments) into that filename,
// and throw an exception with the filename. Makes it easier to
// diagnose. This should be rare; most errors happen only
// during devlpmt of DotNetZip itself, but there are rare
// occasions when they occur in other cases.
string sourceFile = GenerateTempPathname(tmpDir, "cs");
// // debugging: enumerate the resources in this assembly
// Console.WriteLine("Resources in this assembly:");
// foreach (string rsrc in a2.GetManifestResourceNames())
// {
// Console.WriteLine(rsrc);
// }
// Console.WriteLine();
// all the source code is embedded in the DLL as a zip file.
using (ZipFile zip = ZipFile.Read(a2.GetManifestResourceStream("Ionic.Zip.Resources.ZippedResources.zip")))
{
// // debugging: enumerate the files in the embedded zip
// Console.WriteLine("Entries in the embbedded zip:");
// foreach (ZipEntry entry in zip)
// {
// Console.WriteLine(entry.FileName);
// }
// Console.WriteLine();
unpackedResourceDir = GenerateTempPathname(tmpDir, "tmp");
if (String.IsNullOrEmpty(options.IconFile))
{
// Use the ico file that is embedded into the Ionic.Zip
// DLL itself. To do this we must unpack the icon to
// the filesystem, in order to specify it on the cmdline
// of csc.exe. This method will remove the unpacked
// file later.
System.IO.Directory.CreateDirectory(unpackedResourceDir);
ZipEntry e = zip["zippedFile.ico"];
// Must not extract a readonly file - it will be impossible to
// delete later.
if ((e.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
{
e.Attributes ^= FileAttributes.ReadOnly;
}
e.Extract(unpackedResourceDir);
nameOfIconFile = Path.Combine(unpackedResourceDir, "zippedFile.ico");
cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile);
}
else
{
cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", options.IconFile);
}
cp.OutputAssembly = stubExe;
if (options.Flavor == SelfExtractorFlavor.WinFormsApplication)
{
cp.CompilerOptions += " /target:winexe";
}
if (!String.IsNullOrEmpty(options.AdditionalCompilerSwitches))
{
cp.CompilerOptions += " " + options.AdditionalCompilerSwitches;
}
if (String.IsNullOrEmpty(cp.CompilerOptions))
{
cp.CompilerOptions = null;
}
if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0))
{
if (!Directory.Exists(unpackedResourceDir))
{
System.IO.Directory.CreateDirectory(unpackedResourceDir);
}
foreach (string re in settings.CopyThroughResources)
{
string filename = Path.Combine(unpackedResourceDir, re);
ExtractResourceToFile(a2, re, filename);
// add the file into the target assembly as an embedded resource
cp.EmbeddedResources.Add(filename);
}
}
// add the Ionic.Utils.Zip DLL as an embedded resource
cp.EmbeddedResources.Add(a1.Location);
// file header
sb.Append("// " + Path.GetFileName(sourceFile) + "\n")
.Append("// --------------------------------------------\n//\n")
.Append("// This SFX source file was generated by DotNetZip ")
.Append(ZipFile.LibraryVersion.ToString())
.Append("\n// at ")
.Append(System.DateTime.Now.ToString("yyyy MMMM dd HH:mm:ss"))
.Append("\n//\n// --------------------------------------------\n\n\n");
// assembly attributes
if (!String.IsNullOrEmpty(options.Description))
{
sb.Append("[assembly: System.Reflection.AssemblyTitle(\""
+ options.Description.Replace("\"", "")
+ "\")]\n");
}
else
{
sb.Append("[assembly: System.Reflection.AssemblyTitle(\"DotNetZip SFX Archive\")]\n");
}
if (!String.IsNullOrEmpty(options.ProductVersion))
{
sb.Append("[assembly: System.Reflection.AssemblyInformationalVersion(\""
+ options.ProductVersion.Replace("\"", "")
+ "\")]\n");
}
// workitem
string copyright =
(String.IsNullOrEmpty(options.Copyright))
? "Extractor: Copyright � Dino Chiesa 2008-2011"
: options.Copyright.Replace("\"", "");
if (!String.IsNullOrEmpty(options.ProductName))
{
sb.Append("[assembly: System.Reflection.AssemblyProduct(\"")
.Append(options.ProductName.Replace("\"", ""))
.Append("\")]\n");
}
else
{
sb.Append("[assembly: System.Reflection.AssemblyProduct(\"DotNetZip\")]\n");
}
sb.Append("[assembly: System.Reflection.AssemblyCopyright(\"" + copyright + "\")]\n")
.Append(String.Format("[assembly: System.Reflection.AssemblyVersion(\"{0}\")]\n", ZipFile.LibraryVersion.ToString()));
if (options.FileVersion != null)
{
sb.Append(String.Format("[assembly: System.Reflection.AssemblyFileVersion(\"{0}\")]\n",
options.FileVersion.ToString()));
}
sb.Append("\n\n\n");
// Set the default extract location if it is available
string extractLoc = options.DefaultExtractDirectory;
if (extractLoc != null)
{
// remove double-quotes and replace slash with double-slash.
// This, because the value is going to be embedded into a
// cs file as a quoted string, and it needs to be escaped.
extractLoc = extractLoc.Replace("\"", "").Replace("\\", "\\\\");
}
string postExCmdLine = options.PostExtractCommandLine;
if (postExCmdLine != null)
{
postExCmdLine = postExCmdLine.Replace("\\", "\\\\");
postExCmdLine = postExCmdLine.Replace("\"", "\\\"");
}
foreach (string rc in settings.ResourcesToCompile)
{
using (Stream s = zip[rc].OpenReader())
{
if (s == null)
{
throw new ZipException(String.Format("missing resource '{0}'", rc));
}
using (StreamReader sr = new StreamReader(s))
{
while (sr.Peek() >= 0)
{
string line = sr.ReadLine();
if (extractLoc != null)
{
line = line.Replace("@@EXTRACTLOCATION", extractLoc);
}
line = line.Replace("@@REMOVE_AFTER_EXECUTE", options.RemoveUnpackedFilesAfterExecute.ToString());
line = line.Replace("@@QUIET", options.Quiet.ToString());
if (!String.IsNullOrEmpty(options.SfxExeWindowTitle))
{
line = line.Replace("@@SFX_EXE_WINDOW_TITLE", options.SfxExeWindowTitle);
}
line = line.Replace("@@EXTRACT_EXISTING_FILE", ((int)options.ExtractExistingFile).ToString());
if (postExCmdLine != null)
{
line = line.Replace("@@POST_UNPACK_CMD_LINE", postExCmdLine);
}
sb.Append(line).Append("\n");
}
}
sb.Append("\n\n");
}
}
}
string LiteralSource = sb.ToString();
#if DEBUGSFX
// for debugging only
string sourceModule = GenerateTempPathname(tmpDir, "cs");
using (StreamWriter sw = File.CreateText(sourceModule))
{
sw.Write(LiteralSource);
}
Console.WriteLine("source: {0}", sourceModule);
#endif
var cr = csharp.CompileAssemblyFromSource(cp, LiteralSource);
if (cr == null)
{
throw new SfxGenerationException("Cannot compile the extraction logic!");
}
if (Verbose)
{
foreach (string output in cr.Output)
{
StatusMessageTextWriter.WriteLine(output);
}
}
if (cr.Errors.Count != 0)
{
using (TextWriter tw = new StreamWriter(sourceFile))
{
// first, the source we compiled
tw.Write(LiteralSource);
// now, append the compile errors
tw.Write("\n\n\n// ------------------------------------------------------------------\n");
tw.Write("// Errors during compilation: \n//\n");
string p = Path.GetFileName(sourceFile);
foreach (System.CodeDom.Compiler.CompilerError error in cr.Errors)
{
tw.Write(String.Format("// {0}({1},{2}): {3} {4}: {5}\n//\n",
p, // 0
error.Line, // 1
error.Column, // 2
error.IsWarning ? "Warning" : "error", // 3
error.ErrorNumber, // 4
error.ErrorText)); // 5
}
}
throw new SfxGenerationException(String.Format("Errors compiling the extraction logic! {0}", sourceFile));
}
OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor);
// Now, copy the resulting EXE image to the _writestream.
// Because this stub exe is being saved first, the effect will be to
// concatenate the exe and the zip data together.
using (System.IO.Stream input = System.IO.File.OpenRead(stubExe))
{
byte[] buffer = new byte[4000];
int n = 1;
while (n != 0)
{
n = input.Read(buffer, 0, buffer.Length);
if (n != 0)
{
WriteStream.Write(buffer, 0, n);
}
}
}
}
OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive);
}
finally
{
try
{
if (Directory.Exists(unpackedResourceDir))
{
try { Directory.Delete(unpackedResourceDir, true); }
catch (System.IO.IOException exc1)
{
StatusMessageTextWriter.WriteLine("Warning: Exception: {0}", exc1);
}
}
if (File.Exists(stubExe))
{
try { File.Delete(stubExe); }
catch (System.IO.IOException exc1)
{
StatusMessageTextWriter.WriteLine("Warning: Exception: {0}", exc1);
}
}
}
catch (System.IO.IOException) { }
}
return;
}
/// <summary>
/// Saves the ZipFile instance to a self-extracting zip archive.
/// </summary>
///
/// <remarks>
///
/// <para>
/// The generated exe image will execute on any machine that has the .NET Framework 2.0
/// installed on it.
/// </para>
///
/// <para>
/// There are two "flavors" of self-extracting archive. The <c>WinFormsApplication</c>
/// version will pop up a GUI and allow the user to select a target directory into which
/// to extract. There's also a checkbox allowing the user to specify to overwrite
/// existing files, and another checkbox to allow the user to request that Explorer be
/// opened to see the extracted files after extraction. The other flavor is
/// <c>ConsoleApplication</c>. A self-extractor generated with that flavor setting will
/// run from the command line. It accepts command-line options to set the overwrite
/// behavior, and to specify the target extraction directory.
/// </para>
///
/// <para>
/// There are a few temporary files created during the saving to a self-extracting zip.
/// These files are created in the directory pointed to by
/// <see cref="ZipFile.TempFileFolder"/>, which defaults to <see cref="System.IO.Path.GetTempPath"/>.
/// These temporary files are removed upon successful completion of this method.
/// </para>
///
/// <para>
/// When a user runs the SFX, the user's personal directory
/// (<see cref="Environment.SpecialFolder.Personal"/>)
/// will be used as the default extract location.
/// The user who runs the SFX will have the opportunity to change the extract
/// directory before extracting.
/// </para>
///
/// <para>
/// NB: This method is not available in the version of DotNetZip
/// build for the .NET Compact Framework, nor in the "Reduced" DotNEtZip library.
/// </para>
///
/// </remarks>
///
/// <example>
/// <code>
/// string DirectoryPath = "c:\\Documents\\Project7";
/// using (ZipFile zip = new ZipFile())
/// {
/// zip.AddDirectory(DirectoryPath, System.IO.Path.GetFileName(DirectoryPath));
/// zip.Comment = "This will be embedded into a self-extracting console-based exe";
/// zip.SaveSelfExtractor("archive.exe", SelfExtractorFlavor.ConsoleApplication);
/// }
/// </code>
/// <code lang="VB">
/// Dim DirectoryPath As String = "c:\Documents\Project7"
/// Using zip As New ZipFile()
/// zip.AddDirectory(DirectoryPath, System.IO.Path.GetFileName(DirectoryPath))
/// zip.Comment = "This will be embedded into a self-extracting console-based exe"
/// zip.SaveSelfExtractor("archive.exe", SelfExtractorFlavor.ConsoleApplication)
/// End Using
/// </code>
/// </example>
///
/// <param name="exeToGenerate">a pathname, possibly fully qualified, to be created. Typically it will end in an .exe extension.</param>
/// <param name="flavor">Indicates whether a Winforms or Console self-extractor is desired.</param>
public void SaveSelfExtractor(string exeToGenerate, SelfExtractorFlavor flavor)
{
if (File.Exists(exeToGenerate))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate);
}
}
if (!exeToGenerate.EndsWith(".exe"))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension.");
}
}
string TempZipFile = SfxSaveTemporary();
OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive);
if (TempZipFile == null)
{
return; // cancelled
}
// look for myself (ZipFile will be present in the Ionic.Utils.Zip assembly)
Assembly a1 = typeof(ZipFile).Assembly;
//Console.WriteLine("DotNetZip assembly loc: {0}", a1.Location);
Microsoft.CSharp.CSharpCodeProvider csharp = new Microsoft.CSharp.CSharpCodeProvider();
// Perfect opportunity for a linq query, but I cannot use it.
// The DotNetZip library can compile into 2.0, but needs to run on .NET 2.0.
// Using LINQ would break that. Here's what it would look like:
//
// var settings = (from x in SettingsList
// where x.Flavor == flavor
// select x).First();
ExtractorSettings settings = null;
foreach (var x in SettingsList)
{
if (x.Flavor == flavor)
{
settings = x;
break;
}
}
if (settings == null)
{
throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", flavor));
}
// This is the list of referenced assemblies. Ionic.Utils.Zip is needed here.
// Also if it is the winforms (gui) extractor, we need other referenced assemblies.
System.CodeDom.Compiler.CompilerParameters cp = new System.CodeDom.Compiler.CompilerParameters();
cp.ReferencedAssemblies.Add(a1.Location);
if (settings.ReferencedAssemblies != null)
{
foreach (string ra in settings.ReferencedAssemblies)
{
cp.ReferencedAssemblies.Add(ra);
}
}
cp.GenerateInMemory = false;
cp.GenerateExecutable = true;
cp.IncludeDebugInformation = false;
cp.OutputAssembly = exeToGenerate;
Assembly a2 = Assembly.GetExecutingAssembly();
string TempDir = GenerateUniquePathname("tmp", null);
if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0))
{
System.IO.Directory.CreateDirectory(TempDir);
int n = 0;
byte[] bytes = new byte[1024];
foreach (string re in settings.CopyThroughResources)
{
string filename = Path.Combine(TempDir, re);
using (Stream instream = a2.GetManifestResourceStream(re))
{
using (FileStream outstream = File.OpenWrite(filename))
{
do
{
n = instream.Read(bytes, 0, bytes.Length);
outstream.Write(bytes, 0, n);
} while (n > 0);
}
}
// add the embedded resource in our own assembly into the target assembly as an embedded resource
cp.EmbeddedResources.Add(filename);
}
}
// add the zip file as an embedded resource
cp.EmbeddedResources.Add(TempZipFile);
// add the Ionic.Utils.Zip DLL as an embedded resource
cp.EmbeddedResources.Add(a1.Location);
//Console.WriteLine("Resources in this assembly:");
//foreach (string rsrc in a2.GetManifestResourceNames())
//{
// Console.WriteLine(rsrc);
//}
//Console.WriteLine();
//Console.WriteLine("reading source code resources:");
// concatenate all the source code resources into a single module
var sb = new System.Text.StringBuilder();
// set the default extract location if it is available
bool wantCodeReplace = (flavor == SelfExtractorFlavor.WinFormsApplication && _defaultExtractLocation != null);
if (wantCodeReplace)
{
_defaultExtractLocation = _defaultExtractLocation.Replace("\"", "");
}
foreach (string rc in settings.ResourcesToCompile)
{
//Console.WriteLine(" trying to read stream: ({0})", rc);
Stream s = a2.GetManifestResourceStream(rc);
using (StreamReader sr = new StreamReader(s))
{
while (sr.Peek() >= 0)
{
string line = sr.ReadLine();
if (wantCodeReplace)
{
line = line.Replace("@@VALUE", _defaultExtractLocation);
}
sb.Append(line).Append("\n");
}
}
sb.Append("\n\n");
}
string LiteralSource = sb.ToString();
System.CodeDom.Compiler.CompilerResults cr = csharp.CompileAssemblyFromSource(cp, LiteralSource);
if (cr == null)
{
throw new SfxGenerationException("Cannot compile the extraction logic!");
}
if (Verbose)
{
foreach (string output in cr.Output)
{
StatusMessageTextWriter.WriteLine(output);
}
}
if (cr.Errors.Count != 0)
{
throw new SfxGenerationException("Errors compiling the extraction logic!");
}
OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor);
try
{
if (Directory.Exists(TempDir))
{
try { Directory.Delete(TempDir, true); }
catch { }
}
if (File.Exists(TempZipFile))
{
try { File.Delete(TempZipFile); }
catch { }
}
}
catch { }
OnSaveCompleted();
if (Verbose)
{
StatusMessageTextWriter.WriteLine("Created self-extracting zip file {0}.", cr.PathToAssembly);
}
return;
// catch (Exception e1)
// {
// StatusMessageTextWriter.WriteLine("****Exception: " + e1);
// throw;
// }
// return;
}
private void Extract(IConsole console, ExtractorSettings settings)
{
var redirects = new List <(string, string)>();
// Create the output directories if needed
if (!DryRun)
{
PrepareObjectDirectory(settings);
}
//Doc to save the actual content as xml
var xmlDocument = new XmlDocument();
var parentElem = xmlDocument.CreateNode("element", settings.PageType.Replace(".", "-") + "s", "");
//Fire up our Kentico Xperience instance
ConnectionHelper.ConnectionString = settings.ConnectionString;
ValidationHelper.HashStringSalt = settings.HashStringSalt;
CMSApplication.Init();
// Gets an object representing a specific Kentico user
UserInfo user = UserInfoProvider.GetUserInfo(settings.XperienceUser);
// Sets the context of the user
using (new CMSActionContext(user))
{
var wc = new WebClient();
//Get me all published pages of the specified page type on the site
var posts = DocumentHelper.GetDocuments(settings.PageType)
.Columns(settings.PageTypeColumns)
.Path(settings.RootNodeAliasPath, PathTypeEnum.Children)
.PublishedVersion()
.Published()
.TopN(settings.TopN)
.OrderBy(settings.OrderByColumns);
console.WriteLine($"{posts.Count} nodes found.");
console.WriteLine("Exporting node data...");
int nodeOrder = 1;
foreach (TreeNode post in posts)
{
//Create a new AgilityPack doc for working with the page
HtmlDocument doc = new HtmlDocument();
doc.OptionOutputAsXml = false; //if true adds a weird span tag
doc.DocumentNode.AppendChild(HtmlNode.CreateNode("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
doc.DocumentNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine));
string nodeAliasPath = post.NodeAliasPath;
nodeAliasPath = nodeAliasPath.Split('/').Last(); //clean it up to just the slug
// Add redirect to the list
redirects.Add((post.NodeAliasPath, nodeAliasPath));
// TODO: use FormInfo to determine "DocumentName" field
string nodeTitle = post.DocumentName;
//Create a parent folder path in the data directory to store all xml about this node in a structured way on the file system
string nodeDirPath = Path.Combine(settings.ObjectDirectory, settings.PageType, nodeAliasPath.Replace("/", "-"));
if (!Directory.Exists(nodeDirPath))
{
Directory.CreateDirectory(nodeDirPath);
}
//Create root xml Element of the pageTypeName ex. <cms-blogpost>
// and start saving fields as attributes and child elements in this xml structure
var rootNode = HtmlNode.CreateNode($"<{settings.PageType.Replace( ".", "-" )} />");
rootNode.Attributes.Append("originalAliasPath");
rootNode.SetAttributeValue("originalAliasPath", post.NodeAliasPath);
rootNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine));
var aliasNode = HtmlNode.CreateNode($"<newaliaspath/>");
aliasNode.InnerHtml = nodeAliasPath;
rootNode.AppendChild(aliasNode);
var orderNode = HtmlNode.CreateNode($"<newnodeorder/>");
orderNode.InnerHtml = nodeOrder.ToString();
rootNode.AppendChild(orderNode);
nodeOrder++;
//Kentico dynamic properties or "With the Coupled Columns of the page type"
//Iterate through all of the fields of a document in order to serialize them to xml
foreach (var dprop in post.Properties)
{
if (Array.IndexOf(settings.PageTypeColumns, dprop) > 0)
{
//Create xml Element that represents each field ex. <blogpostsummary>data</blogpostsummary>
var node = HtmlNode.CreateNode($"<{dprop} />");
var data = post.GetValue(dprop, "");
//Special stuff for Mcbeev Blog Post Summary (can be ignored for others)
if (dprop.ToLower().Contains("summary"))
{
//Remove wrapped p tag on Mcbeev Blog Post Summary
HtmlDocument docSummary = new HtmlDocument();
docSummary.LoadHtml(data);
var newSummaryNode = HtmlNode.CreateNode($"<newblogpostsummary/>");
try
{
if (docSummary.DocumentNode.SelectSingleNode("//p") != null)
{
newSummaryNode.InnerHtml = docSummary.DocumentNode.SelectSingleNode("//p").InnerText;
}
else
{
newSummaryNode.InnerHtml = docSummary.DocumentNode.InnerText;
}
}
catch (Exception ex)
{
//todo
console.WriteLine(ex.Message);
}
rootNode.AppendChild(newSummaryNode);
rootNode.AppendChild(HtmlNode.CreateNode(Environment.NewLine));
}
//This is WYSIWYG content field most likely so try to resolve things as Kentico would
if (dprop.ToLower().Contains("title") || dprop.ToLower().Contains("body") || dprop.ToLower().Contains("content") || dprop.ToLower().Contains("summary"))
{
//Macro resolve inside of the content (handle K# macros in the text)
data = MacroResolver.Resolve(data);
//Url resolve inside of the content (handle relative urls into absolute urls) important!
data = HTMLHelper.ResolveUrls(data, settings.AbsoluteSiteName, true);
//Widget identifier (try to save a list of all Portal Inline Widgets that no longer work in MVC)
if (data.Contains("CMSInlineControl"))
{
console.WriteLine("Found a widget in this content.");
// TODO Make this a real log
// //Render the value as html to HP NodedataRootDirectoryName
// HtmlDocument docWidget = new HtmlDocument();
// docWidget.LoadHtml(data);
// var widgets = docWidget.DocumentNode.SelectNodes("//object[contains(@type, 'Widget')]");
// foreach (var w in widgets)
// {
// console.WriteLine(w.OuterHtml);
// }
}
//Media library identifier
if (data.Contains("<img "))
{
HtmlDocument docImg = new HtmlDocument();
docImg.LoadHtml(data);
var ImageURLs = docImg.DocumentNode.Descendants("img")
.Select(e => e.GetAttributeValue("src", null))
.Where(s => !String.IsNullOrEmpty(s));
foreach (var oldImageURL in ImageURLs)
{
//Skip mp3s for now
if (oldImageURL.EndsWith(".mp3") || oldImageURL.Contains(".mp3?"))
{
continue;
}
console.WriteLine($"trying to save {oldImageURL} ...");
var safeName = oldImageURL.Split(new char[] { '?' })[0];
safeName = safeName.Replace(".aspx", "");
//Save the images locally if we can download from the "real" running site
try
{
wc.DownloadFile(oldImageURL, Path.Combine(nodeDirPath, Path.GetFileName(safeName)));
}
catch (Exception ex)
{
console.WriteLine(ex.Message);
}
//rewrite the image path back in the data node
// TODO - make this a config parameter and not hard coded to Mcbeev.com
data = data.Replace(oldImageURL, "/MBV/media/blog/" + nodeAliasPath.Replace("/", "-") + "/" + oldImageURL.Split(new char[] { '/' }).Last());
//replace the odd .png.aspx thing
data = data.Replace(".png.aspx", ".png");
}
}
//Use HTMLAgility Pack to fix non closed image tags
HtmlNode.ElementsFlags["img"] = HtmlElementFlag.Closed;
HtmlDocument docContent = new HtmlDocument();
docContent.LoadHtml(data);
data = $"<![CDATA[{docContent.DocumentNode.OuterHtml}]]>";
}
//Most likely image only field and not a WYSIWYG field
// TODO - is there a better way to check for this?
if ((data.Length > 2) && (data.StartsWith("~/")))
{
data = data.Replace("~/", settings.AbsoluteSiteName);
console.WriteLine($"trying to save {data} ...");
var safeName = data.Split(new char[] { '?' })[0];
safeName = safeName.Replace(".aspx", "");
wc.DownloadFile(data, Path.Combine(nodeDirPath, Path.GetFileName(safeName)));
//rewrite the image path back in the data node
safeName = "/MBV/media/blog/" + nodeAliasPath.Replace("/", "-") + "/" + safeName.Split(new char[] { '/' }).Last();
data = $"<![CDATA[{safeName}]]>";
}
node.AppendChild(doc.CreateTextNode(data));
rootNode.AppendChild(node);
rootNode.AppendChild(HtmlNode.CreateNode("\r\n"));
}
}
doc.DocumentNode.AppendChild(rootNode);
//Save the single node to its own XML document in its directory
doc.Save($@"{nodeDirPath}\{nodeAliasPath.Replace( "/", "-" )}.xml");
//Chop off the header to prepare for bundling
doc.DocumentNode.ChildNodes[0].Remove();
//add this node to all nodes in main XML doc
parentElem.InnerXml += doc.DocumentNode.OuterHtml;
xmlDocument.AppendChild(parentElem);
}
if (!DryRun)
{
WriteRedirects(redirects, settings);
xmlDocument.Save(Path.Combine(settings.ObjectDirectory, settings.PageType, settings.AllNodesFilename));
console.WriteLine($"Extraction to {settings.ObjectDirectory} succeeded");
}
else
{
console.WriteLine("Dry run succeeded");
}
}
}
private void _SaveSfxStub(string exeToGenerate, SelfExtractorFlavor flavor, string defaultExtractLocation, string postExtractCmdLine, string nameOfIconFile)
{
bool removeIconFile = false;
string StubExe = null;
string TempDir = null;
try
{
if (File.Exists(exeToGenerate))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("The existing file ({0}) will be overwritten.", exeToGenerate);
}
}
if (!exeToGenerate.EndsWith(".exe"))
{
if (Verbose)
{
StatusMessageTextWriter.WriteLine("Warning: The generated self-extracting file will not have an .exe extension.");
}
}
StubExe = GenerateTempPathname("exe", null);
// get the Ionic.Zip assembly
Assembly a1 = typeof(ZipFile).Assembly;
Microsoft.CSharp.CSharpCodeProvider csharp = new Microsoft.CSharp.CSharpCodeProvider();
// Perfect opportunity for a linq query, but I cannot use it.
// The DotNetZip library can compile into 2.0, but needs to run on .NET 2.0.
// Using LINQ would break that. Here's what it would look like:
//
// var settings = (from x in SettingsList
// where x.Flavor == flavor
// select x).First();
ExtractorSettings settings = null;
foreach (var x in SettingsList)
{
if (x.Flavor == flavor)
{
settings = x;
break;
}
}
if (settings == null)
{
throw new BadStateException(String.Format("While saving a Self-Extracting Zip, Cannot find that flavor ({0})?", flavor));
}
// This is the list of referenced assemblies. Ionic.Zip is needed here.
// Also if it is the winforms (gui) extractor, we need other referenced assemblies,
// like System.Windows.Forms.dll, etc.
System.CodeDom.Compiler.CompilerParameters cp = new System.CodeDom.Compiler.CompilerParameters();
cp.ReferencedAssemblies.Add(a1.Location);
if (settings.ReferencedAssemblies != null)
{
foreach (string ra in settings.ReferencedAssemblies)
{
cp.ReferencedAssemblies.Add(ra);
}
}
cp.GenerateInMemory = false;
cp.GenerateExecutable = true;
cp.IncludeDebugInformation = false;
cp.CompilerOptions = "";
Assembly a2 = Assembly.GetExecutingAssembly();
if (nameOfIconFile == null)
{
removeIconFile = true;
nameOfIconFile = GenerateTempPathname("ico", null);
ExtractResourceToFile(a2, "Ionic.Zip.Resources.zippedFile.ico", nameOfIconFile);
cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile);
}
else if (nameOfIconFile != "")
{
cp.CompilerOptions += String.Format("/win32icon:\"{0}\"", nameOfIconFile);
}
//cp.IncludeDebugInformation = true;
cp.OutputAssembly = StubExe;
if (flavor == SelfExtractorFlavor.WinFormsApplication)
{
cp.CompilerOptions += " /target:winexe";
}
if (cp.CompilerOptions == "")
{
cp.CompilerOptions = null;
}
TempDir = GenerateTempPathname("tmp", null);
if ((settings.CopyThroughResources != null) && (settings.CopyThroughResources.Count != 0))
{
System.IO.Directory.CreateDirectory(TempDir);
foreach (string re in settings.CopyThroughResources)
{
string filename = Path.Combine(TempDir, re);
ExtractResourceToFile(a2, re, filename);
// add the file into the target assembly as an embedded resource
cp.EmbeddedResources.Add(filename);
}
}
// add the Ionic.Utils.Zip DLL as an embedded resource
cp.EmbeddedResources.Add(a1.Location);
//Console.WriteLine("Resources in this assembly:");
//foreach (string rsrc in a2.GetManifestResourceNames())
//{
// Console.WriteLine(rsrc);
//}
//Console.WriteLine();
//Console.WriteLine("reading source code resources:");
// concatenate all the source code resources into a single module
var sb = new System.Text.StringBuilder();
// assembly attributes
sb.Append("[assembly: System.Reflection.AssemblyTitle(\"DotNetZip SFX Archive\")]\n");
sb.Append("[assembly: System.Reflection.AssemblyProduct(\"ZipLibrary\")]\n");
sb.Append("[assembly: System.Reflection.AssemblyCopyright(\"Copyright © Dino Chiesa 2008, 2009\")]\n");
sb.Append(String.Format("[assembly: System.Reflection.AssemblyVersion(\"{0}\")]\n\n", ZipFile.LibraryVersion.ToString()));
// Set the default extract location if it is available, and if supported.
bool haveLocation = (defaultExtractLocation != null);
if (haveLocation)
{
defaultExtractLocation = defaultExtractLocation.Replace("\"", "").Replace("\\", "\\\\");
}
foreach (string rc in settings.ResourcesToCompile)
{
//Console.WriteLine(" trying to read stream: ({0})", rc);
Stream s = a2.GetManifestResourceStream(rc);
if (s == null)
{
throw new ZipException(String.Format("missing resource '{0}'", rc));
}
using (StreamReader sr = new StreamReader(s))
{
while (sr.Peek() >= 0)
{
string line = sr.ReadLine();
if (haveLocation)
{
line = line.Replace("@@EXTRACTLOCATION", defaultExtractLocation);
}
if (postExtractCmdLine != null)
{
line = line.Replace("@@POST_UNPACK_CMD_LINE", postExtractCmdLine.Replace("\\", "\\\\"));
}
sb.Append(line).Append("\n");
}
}
sb.Append("\n\n");
}
string LiteralSource = sb.ToString();
#if DEBUGSFX
// for debugging only
string sourceModule = GenerateTempPathname("cs", null);
using (StreamWriter sw = File.CreateText(sourceModule))
{
sw.Write(LiteralSource);
}
Console.WriteLine("source: {0}", sourceModule);
#endif
System.CodeDom.Compiler.CompilerResults cr = csharp.CompileAssemblyFromSource(cp, LiteralSource);
if (cr == null)
{
throw new SfxGenerationException("Cannot compile the extraction logic!");
}
if (Verbose)
{
foreach (string output in cr.Output)
{
StatusMessageTextWriter.WriteLine(output);
}
}
if (cr.Errors.Count != 0)
{
//Console.ReadLine();
string sourcefile = GenerateTempPathname("cs", null);
using (TextWriter tw = new StreamWriter(sourcefile))
{
tw.Write(LiteralSource);
}
throw new SfxGenerationException(String.Format("Errors compiling the extraction logic! {0}", sourcefile));
}
OnSaveEvent(ZipProgressEventType.Saving_AfterCompileSelfExtractor);
// Now, copy the resulting EXE image to the _writestream.
// Because this stub exe is being saved first, the effect will be to
// concatenate the exe and the zip data together.
using (System.IO.Stream input = System.IO.File.OpenRead(StubExe))
{
byte[] buffer = new byte[4000];
int n = 1;
while (n != 0)
{
n = input.Read(buffer, 0, buffer.Length);
if (n != 0)
{
WriteStream.Write(buffer, 0, n);
}
}
}
OnSaveEvent(ZipProgressEventType.Saving_AfterSaveTempArchive);
}
finally
{
try
{
if (Directory.Exists(TempDir))
{
try { Directory.Delete(TempDir, true); }
catch { }
}
if (File.Exists(StubExe))
{
try { File.Delete(StubExe); }
catch { }
}
if (removeIconFile && File.Exists(nameOfIconFile))
{
try { File.Delete(nameOfIconFile); }
catch { }
}
}
catch { }
}
return;
}
private async Task WriteConfigAsync(ExtractorSettings settings)
{
using var file = new FileStream(Output, FileMode.Create);
await JsonSerializer.SerializeAsync(file, settings, new JsonSerializerOptions { WriteIndented = true });
}
public static void Validate(this ExtractorSettings settings)
{
var context = new ValidationContext(settings, serviceProvider: null, items: null);
Validator.ValidateObject(settings, context, true);
}
public StartTerminalsCreator(ExtractorSettings settings)
{
_settings = settings;
}
