public ActionResult Index()
{
var doc = new XDocument(new XDeclaration("1.0", "utf-8", null));
var indexElement = new XElement(SitemapXmlNamespace + "sitemapindex");
foreach (var sitemapData in _sitemapRepository.GetAllSitemapData())
{
var sitemapElement = new XElement(
SitemapXmlNamespace + "sitemap",
new XElement(SitemapXmlNamespace + "loc", _sitemapRepository.GetSitemapUrl(sitemapData))
);
indexElement.Add(sitemapElement);
}
doc.Add(indexElement);
CompressionHandler.ChooseSuitableCompression(Request.Headers, Response);
byte[] sitemapIndexData;
using (var ms = new MemoryStream())
{
var xtw = new XmlTextWriter(ms, Encoding.UTF8);
doc.Save(xtw);
xtw.Flush();
sitemapIndexData = ms.ToArray();
}
return(new FileContentResult(sitemapIndexData, "text/xml"));
}
public HomeController(
AppSetting appSetting,
CompressionHandler compressionHandler,
IWebHostEnvironment webHostEnvironment)
{
_appSetting = appSetting;
_compressionHandler = compressionHandler;
_webHostEnvironment = webHostEnvironment;
}
public bool CompressCheckedTraceFiles(int filesToKeep, string sevenZipFileName)
{
bool success = true;
OutputHandler.WriteToLog(string.Format("Compressing Trace Files after import. Number of Trace Files to compress: {0}", traceFileListView.CheckedItems.Count));
try
{
GenericHelper.WriteFile(sevenZipFileName, SQLEventAnalyzer.Properties.Resources._7za);
}
catch (Exception ex)
{
OutputHandler.WriteToLog(string.Format("Error writing 7za.exe in CompressCheckedTraceFiles: \"{0}\"", ex.Message));
success = false;
}
if (success)
{
for (int i = 0; i < traceFileListView.CheckedItems.Count; i++)
{
string fileName = string.Format(@"{0}\{1}", _traceFileDir, traceFileListView.CheckedItems[i].SubItems[2].Text);
try
{
CompressionHandler.CompressFile(fileName, string.Format(@"{0}\{1}.7z", _traceFileDir, Path.GetFileNameWithoutExtension(fileName)), sevenZipFileName);
}
catch (Exception ex)
{
success = false;
string text = "Error compressing Trace File.\r\n\r\n{0}";
if (ConfigHandler.UseTranslation)
{
text = Translator.GetText("errorCompressingTrace");
}
OutputHandler.Show(string.Format(text, ex.Message), GenericHelper.ApplicationName, MessageBoxButtons.OK, MessageBoxIcon.Warning);
}
}
if (filesToKeep > 0)
{
DeleteOldCompressedFiles(filesToKeep, _traceFileDir);
}
try
{
GenericHelper.DeleteFile(sevenZipFileName);
}
catch
{
}
}
return(success);
}
public void DoesNotChangeContentEncodingIfNoSuitableEncodingWasFound()
{
var res = CreateResponseBase();
var emptyHeaders = new NameValueCollection();
CompressionHandler.ChooseSuitableCompression(emptyHeaders, res);
Assert.True(res.Headers.Get("Content-Encoding") == null);
}
public void ChoosesMostSuitableEncoding()
{
var res = CreateResponseBase();
var headers = new NameValueCollection();
headers.Add(CompressionHandler.ACCEPT_ENCODING_HEADER, "gzip;q=0.3,deflate;q=0.8,foobar;q=0.9");
CompressionHandler.ChooseSuitableCompression(headers, res);
var encoding = res.Headers.Get(CompressionHandler.CONTENT_ENCODING_HEADER);
Assert.Equal("deflate", encoding);
}
public void ChangesContentEncodingIfSuitableEncodingWasFound()
{
var res = CreateResponseBase();
var headers = new NameValueCollection();
headers.Add(CompressionHandler.ACCEPT_ENCODING_HEADER, "gzip");
CompressionHandler.ChooseSuitableCompression(headers, res);
var encoding = res.Headers.Get(CompressionHandler.CONTENT_ENCODING_HEADER);
Assert.True(encoding != null);
Assert.Equal("gzip", encoding);
}
private string ParseCompressedStream(string fileList, string compressedStream)
{
string path;
var FileList = ExtractWholeFile(fileList);
if (FileList == null)
{
return(null);
}
using (var CompressedStream = ExtractWholeFile(compressedStream))
{
path = CompressionHandler.DecompressFiles(FileList, CompressedStream, CompType);
}
return(path);
}
public void DoesNotChangeFilterIfNoSuitableEncodingWasFound()
{
// Arrange
var res = CreateResponseBase();
var emptyHeaders = new NameValueCollection();
var beforeFilter = res.Filter;
// Act
CompressionHandler.ChooseSuitableCompression(emptyHeaders, res);
// Assert
var afterFilter = res.Filter;
Assert.Equal(beforeFilter, afterFilter);
}
private string ParseCompressedStream(string dataInfo, string dataCompressed)
{
var infoStream = ExtractWholeFile(dataInfo);
if (infoStream == null)
{
return(null);
}
var compressedStream = ExtractWholeFile(dataCompressed);
var path = CompressionHandler.DecompressFiles(infoStream, compressedStream, Compression);
infoStream.Close();
compressedStream.Close();
return(path);
}
public void TestZipCompression()
{
var inputText = GetDummyString();
var bytes = Encoding.UTF8.GetBytes(inputText);
using var inputStream = new MemoryStream(bytes, true);
using var compressedStream = CompressionHandler.ZipCompress(inputStream, System.IO.Compression.CompressionLevel.Optimal);
using var decompressedStream = CompressionHandler.ZipDecompress(compressedStream, inputStream.Length);
var outputBytes = new byte[decompressedStream.Length];
decompressedStream.Read(outputBytes, 0, outputBytes.Length);
var outputString = Encoding.UTF8.GetString(outputBytes, 0, outputBytes.Length);
Assert.Equal(inputText, outputString);
}
public void TestWriteStreamToZip()
{
using (var zipStream = new ZipOutputStream(File.Open("hello.zip", FileMode.CreateNew)))
using (var bw = new BinaryWriter(zipStream))
{
var ze = new ZipEntry(HelloFile);
zipStream.PutNextEntry(ze);
var fs = File.OpenRead(HelloFile);
CompressionHandler.WriteStreamToZip(bw, fs);
bw.Flush();
fs.Close();
}
using (var zf = new ZipFile(File.OpenRead("hello.zip")))
using (var fs = new FileStream("hello_out.txt", FileMode.Create))
{
zf.GetInputStream(zf.GetEntry("hello.txt")).CopyTo(fs);
}
string hash1;
string hash2;
using (var md5 = MD5.Create())
{
using (var stream = File.OpenRead("hello.txt"))
{
var hash = md5.ComputeHash(stream);
hash1 = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
using (var stream = File.OpenRead("hello_out.txt"))
{
var hash = md5.ComputeHash(stream);
hash2 = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
}
var f1 = new FileInfo("hello.txt");
var f2 = new FileInfo("hello_out.txt");
Assert.AreEqual(hash1, hash2);
Assert.AreEqual(f1.Length, f2.Length);
}
public void TestSevenZipCompression()
{
var inputText = GetDummyString();
var bytes = Encoding.UTF8.GetBytes(inputText);
using var inputStream = new MemoryStream(bytes, false);
using var compressedStream = CompressionHandler.SevenZipCompress(inputStream, SevenZipCompressionLevel.Medium);
using var decompressedStream = CompressionHandler.SevenZipDecompress(compressedStream, inputStream.Length);
var outputBytes = new byte[decompressedStream.Length];
decompressedStream.Read(outputBytes, 0, outputBytes.Length);
var outputString = Encoding.UTF8.GetString(outputBytes, 0, outputBytes.Length);
Assert.Equal(inputText, outputString);
}
public void CreatePipelineWithHttpMessageHandlerInput()
{
using (AuthenticationHandler authenticationHandler = (AuthenticationHandler)GraphClientFactory.CreatePipeline(handlers, this.testHttpMessageHandler))
using (CompressionHandler compressionHandler = (CompressionHandler)authenticationHandler.InnerHandler)
using (RetryHandler retryHandler = (RetryHandler)compressionHandler.InnerHandler)
using (RedirectHandler redirectHandler = (RedirectHandler)retryHandler.InnerHandler)
using (MockRedirectHandler innerMost = (MockRedirectHandler)redirectHandler.InnerHandler)
{
Assert.NotNull(authenticationHandler);
Assert.NotNull(compressionHandler);
Assert.NotNull(retryHandler);
Assert.NotNull(redirectHandler);
Assert.NotNull(innerMost);
Assert.IsType <AuthenticationHandler>(authenticationHandler);
Assert.IsType <CompressionHandler>(compressionHandler);
Assert.IsType <RetryHandler>(retryHandler);
Assert.IsType <RedirectHandler>(redirectHandler);
Assert.IsType <MockRedirectHandler>(innerMost);
}
}
public void Should_CreatePipeline_Without_HttpMessageHandlerInput()
{
using (AuthenticationHandler authenticationHandler = (AuthenticationHandler)GraphClientFactory.CreatePipeline(handlers))
using (CompressionHandler compressionHandler = (CompressionHandler)authenticationHandler.InnerHandler)
using (RetryHandler retryHandler = (RetryHandler)compressionHandler.InnerHandler)
using (RedirectHandler redirectHandler = (RedirectHandler)retryHandler.InnerHandler)
using (HttpMessageHandler innerMost = redirectHandler.InnerHandler)
{
Assert.NotNull(authenticationHandler);
Assert.NotNull(compressionHandler);
Assert.NotNull(retryHandler);
Assert.NotNull(redirectHandler);
Assert.NotNull(innerMost);
Assert.IsType <AuthenticationHandler>(authenticationHandler);
Assert.IsType <CompressionHandler>(compressionHandler);
Assert.IsType <RetryHandler>(retryHandler);
Assert.IsType <RedirectHandler>(redirectHandler);
Assert.True(innerMost is HttpMessageHandler);
}
}
private static bool CompressStatisticsDirectory(string tempPath, string archiveName, string sevenZipFileName)
{
bool success = true;
try
{
GenericHelper.WriteFile(sevenZipFileName, SQLEventAnalyzer.Properties.Resources._7za);
}
catch (Exception ex)
{
OutputHandler.WriteToLog(string.Format("Error writing 7za.exe in CompressStatisticsDirectory: \"{0}\"", ex.Message));
success = false;
}
if (success)
{
try
{
CompressionHandler.CompressDirectory(tempPath, archiveName, sevenZipFileName);
}
catch (Exception ex)
{
OutputHandler.WriteToLog(string.Format("Error creating statistics archive: \"{0}\"", ex.Message));
success = false;
}
}
if (success)
{
try
{
GenericHelper.DeleteFile(sevenZipFileName);
}
catch
{
}
}
return(success);
}
public ActionResult Index()
{
SitemapData sitemapData = _sitemapRepository.GetSitemapData(Request.Url.ToString());
if (sitemapData == null)
{
Log.Error("Xml sitemap data not found!");
return(new HttpNotFoundResult());
}
if (sitemapData.Data == null || (SitemapSettings.Instance.EnableRealtimeSitemap))
{
if (!GetSitemapData(sitemapData))
{
Log.Error("Xml sitemap data not found!");
return(new HttpNotFoundResult());
}
}
CompressionHandler.ChooseSuitableCompression(Request.Headers, Response);
return(new FileContentResult(sitemapData.Data, "text/xml; charset=utf-8"));
}
public void TestCompression()
{
var files = new List <string> {
HelloFile
};
var filesPath = new List <string> {
""
};
CompressionHandler.CompressFiles(files, filesPath, out var dataCompressed, out var dataInfo,
CompressionType.Zip, CompressionLevel.Medium);
Assert.IsNotNull(dataCompressed);
Assert.IsNotNull(dataInfo);
using (var zipStream = new ZipOutputStream(File.Open("hello.zip", FileMode.CreateNew)))
using (var bw = new BinaryWriter(zipStream))
{
zipStream.SetLevel(0);
var ze = new ZipEntry("hello");
zipStream.PutNextEntry(ze);
CompressionHandler.WriteStreamToZip(bw, dataCompressed);
bw.Flush();
zipStream.SetLevel(ZipHandler.GetCompressionLevel(CompressionLevel.Medium));
ze = new ZipEntry("hello.crc");
zipStream.PutNextEntry(ze);
CompressionHandler.WriteStreamToZip(bw, dataInfo);
bw.Flush();
}
Assert.IsTrue(File.Exists("hello.zip"));
dataCompressed.Close();
dataInfo.Close();
}
public CompressionTest()
{
_compressionHandler = ServiceLocator.Current.GetInstance <CompressionHandler>();
}
static void Main(string[] args)
{
var json = File.ReadAllLines(@"C:\Users\Ayende\Downloads\RadnomUsers\RadnomUsers.json");
//var trainer = new CompressionTrainer();
//for (int i = 0; i < json.Length/100; i++)
//{
// trainer.TrainOn(json[i*100]);
//}
//var compressionHandler = trainer.CreateHandler();
//using (var file = File.Create("compression.dic"))
//{
// compressionHandler.Save(file);
// file.Flush();
//}
//return;
CompressionHandler compressionHandler;
using (var file = File.OpenRead("compression.dic"))
{
compressionHandler = CompressionHandler.Load(file);
}
var items = new List <byte[]>();
int size = 0;
int compressedSize = 0;
var ms = new MemoryStream();
int count = 0;
foreach (var s in json)
{
if (count++ % 1000 == 0)
{
Console.WriteLine(count);
}
size += s.Length;
ms.SetLength(0);
compressedSize += compressionHandler.Compress(s, ms);
items.Add(ms.ToArray());
}
//Console.WriteLine(size);
//Console.WriteLine(compressedSize);
for (int i = 0; i < items.Count; i++)
{
if (count-- % 1000 == 0)
{
Console.WriteLine(count);
}
var memoryStream = new MemoryStream(items[i]);
var decompress = compressionHandler.Decompress(memoryStream);
Debug.Assert(decompress.Length == json[i].Length);
for (int j = 0; j < decompress.Length; j++)
{
if (((char)decompress[j] != json[i][j]))
{
Console.WriteLine("BAD");
Console.WriteLine(json[i]);
Console.WriteLine(Encoding.UTF8.GetString(decompress));
break;
}
}
}
Console.WriteLine("All good");
}
public CompressionHandlerTests()
{
this.testHttpMessageHandler = new MockRedirectHandler();
this.compressionHandler = new CompressionHandler(this.testHttpMessageHandler);
this.invoker = new HttpMessageInvoker(this.compressionHandler);
}
public static void CreateOMOD(OMODCreationOptions ops, string omodFileName)
{
Utils.Info($"Creating OMOD to {omodFileName}");
if (File.Exists(omodFileName))
{
throw new OMODFrameworkException($"The provided omodFileName {omodFileName} already exists!");
}
using (var zipStream = new ZipOutputStream(File.Open(omodFileName, FileMode.CreateNew)))
using (var omodStream = new BinaryWriter(zipStream))
{
ZipEntry ze;
zipStream.SetLevel(ZipHandler.GetCompressionLevel(ops.OMODCompressionLevel));
if (!string.IsNullOrWhiteSpace(ops.Readme))
{
Utils.Debug("Writing readme to OMOD");
ze = new ZipEntry("readme");
zipStream.PutNextEntry(ze);
omodStream.Write(ops.Readme);
omodStream.Flush();
}
if (!string.IsNullOrWhiteSpace(ops.Script))
{
Utils.Debug("Writing script to OMOD");
ze = new ZipEntry("script");
zipStream.PutNextEntry(ze);
omodStream.Write(ops.Script);
omodStream.Flush();
}
if (!string.IsNullOrWhiteSpace(ops.Image))
{
Utils.Debug("Writing image to OMOD");
ze = new ZipEntry("image");
zipStream.PutNextEntry(ze);
try
{
using (var fs = File.OpenRead(ops.Image))
{
CompressionHandler.WriteStreamToZip(omodStream, fs);
omodStream.Flush();
}
}
catch (Exception e)
{
throw new OMODFrameworkException($"There was an exception while trying to read the image at {ops.Image}!\n{e}");
}
}
Utils.Debug("Writing config to OMOD");
ze = new ZipEntry("config");
zipStream.PutNextEntry(ze);
omodStream.Write(Framework.Settings.CurrentOMODVersion);
omodStream.Write(ops.Name);
omodStream.Write(ops.MajorVersion);
omodStream.Write(ops.MinorVersion);
omodStream.Write(ops.Author);
omodStream.Write(ops.Email);
omodStream.Write(ops.Website);
omodStream.Write(ops.Description);
omodStream.Write(DateTime.Now.ToBinary());
omodStream.Write((byte)ops.CompressionType);
omodStream.Write(ops.BuildVersion);
omodStream.Flush();
FileStream dataCompressed;
Stream dataInfo;
if (ops.ESPs.Count > 0)
{
Utils.Debug("Writing plugins.crc to OMOD");
//TODO: find out why OBMM calls GC.Collect here
ze = new ZipEntry("plugins.crc");
zipStream.PutNextEntry(ze);
CompressionHandler.CompressFiles(ops.ESPs, ops.ESPPaths, out dataCompressed, out dataInfo,
ops.CompressionType, ops.DataFileCompressionLevel);
CompressionHandler.WriteStreamToZip(omodStream, dataInfo);
omodStream.Flush();
zipStream.SetLevel(0);
Utils.Debug("Writing plugins to OMOD");
ze = new ZipEntry("plugins");
zipStream.PutNextEntry(ze);
CompressionHandler.WriteStreamToZip(omodStream, dataCompressed);
omodStream.Flush();
zipStream.SetLevel(ZipHandler.GetCompressionLevel(ops.OMODCompressionLevel));
dataCompressed.Close();
dataInfo.Close();
}
if (ops.DataFiles.Count > 0)
{
Utils.Debug("Writing data.crc to OMOD");
//TODO: find out why OBMM calls GC.Collect here
ze = new ZipEntry("data.crc");
zipStream.PutNextEntry(ze);
CompressionHandler.CompressFiles(ops.DataFiles, ops.DataFilePaths, out dataCompressed, out dataInfo,
ops.CompressionType, ops.DataFileCompressionLevel);
CompressionHandler.WriteStreamToZip(omodStream, dataInfo);
omodStream.Flush();
zipStream.SetLevel(0);
Utils.Debug("Writing data to OMOD");
ze = new ZipEntry("data");
zipStream.PutNextEntry(ze);
CompressionHandler.WriteStreamToZip(omodStream, dataCompressed);
omodStream.Flush();
zipStream.SetLevel(ZipHandler.GetCompressionLevel(ops.OMODCompressionLevel));
dataCompressed.Close();
dataInfo.Close();
}
zipStream.Finish();
}
Utils.Info("Finished OMOD creation");
}
public OMOD(string path)
{
Utils.Info($"Loading OMOD from {path}...");
if (!File.Exists(path))
{
throw new OMODFrameworkException($"The provided file at {path} does not exists!");
}
FilePath = Path.GetDirectoryName(path);
FileName = Path.GetFileName(path);
LowerFileName = FileName.ToLower();
Utils.Debug("Parsing config file from OMOD");
using (var configStream = ExtractWholeFile("config"))
{
if (configStream == null)
{
throw new OMODFrameworkException($"Could not find the configuration data for {FileName} !");
}
using (var br = new BinaryReader(configStream))
{
var fileVersion = br.ReadByte();
if (fileVersion > Framework.Settings.CurrentOMODVersion && !Framework.Settings.IgnoreVersionCheck)
{
throw new OMODFrameworkException($"{FileName} was created with a newer version of OBMM and could not be loaded!");
}
ModName = br.ReadString();
MajorVersion = br.ReadInt32();
MinorVersion = br.ReadInt32();
Author = br.ReadString();
Email = br.ReadString();
Website = br.ReadString();
Description = br.ReadString();
if (fileVersion >= 2)
{
CreationTime = DateTime.FromBinary(br.ReadInt64());
}
else
{
var sCreationTime = br.ReadString();
if (!DateTime.TryParseExact(sCreationTime, "dd/MM/yyyy HH:mm", null,
System.Globalization.DateTimeStyles.NoCurrentDateDefault, out CreationTime))
{
CreationTime = new DateTime(2006, 1, 1);
}
}
if (Description == "")
{
Description = "No description";
}
Compression = (CompressionType)br.ReadByte();
if (fileVersion >= 1)
{
BuildVersion = br.ReadInt32();
}
else
{
BuildVersion = -1;
}
AllPlugins = GetPluginSet();
AllDataFiles = GetDataSet();
AllPlugins.Do(p =>
{
if (!Utils.IsSafeFileName(p))
{
throw new OMODFrameworkException($"File {FileName} has been modified and will not be loaded!");
}
});
AllDataFiles.Do(d =>
{
if (!Utils.IsSafeFileName(d.FileName))
{
throw new OMODFrameworkException($"File {FileName} has been modified and will not be loaded!");
}
});
CRC = CompressionHandler.CRC(FullFilePath);
}
Close();
Utils.Debug("Finished parsing of the config file");
}
}
/// <summary>
/// Create a new OMOD as a <see cref="MemoryStream"/>.
/// </summary>
/// <param name="options">Creation options to use.</param>
/// <returns></returns>
public static MemoryStream CreateOMOD(OMODCreationOptions options)
{
var ms = new MemoryStream();
using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true, Encoding.UTF8))
{
void WriteStringToArchive(string name, string content)
{
var entry = archive.CreateEntry(name, options.OMODCompressionLevel);
using var entryStream = entry.Open();
using var bw = new BinaryWriter(entryStream);
bw.Write(content);
}
if (options.Readme != null)
{
WriteStringToArchive("readme", options.Readme);
}
if (options.Script != null)
{
WriteStringToArchive("script", $"{(char) options.ScriptType}{options.Script}");
}
if (options.Image != null)
{
var entry = archive.CreateEntry("image", options.OMODCompressionLevel);
using var entryStream = entry.Open();
options.Image.Save(entryStream, options.Image.RawFormat);
}
{
var config = archive.CreateEntry("config", options.OMODCompressionLevel);
using var configStream = config.Open();
using var bw = new BinaryWriter(configStream);
bw.Write((byte)4);
bw.Write(options.Name);
bw.Write(options.Version.Major);
bw.Write(options.Version.Minor);
bw.Write(options.Author);
bw.Write(options.Email);
bw.Write(options.Website);
bw.Write(options.Description);
bw.Write(DateTime.Now.ToBinary());
bw.Write((byte)options.CompressionType);
bw.Write(options.Version.Build);
}
void WriteFilesToArchive(string name, IEnumerable <OMODCreationFile> files)
{
CompressionHandler.CompressFiles(files, options.CompressionType,
options.SevenZipCompressionLevel, options.ZipCompressionLevel, out var crcStream,
out var compressedStream);
//Entries cannot be created while previously created entries are still open
{
var crcEntry = archive.CreateEntry($"{name}.crc", options.OMODCompressionLevel);
using var crcEntryStream = crcEntry.Open();
crcStream.CopyTo(crcEntryStream);
crcStream.Dispose();
}
{
var dataEntry = archive.CreateEntry(name, CompressionLevel.NoCompression);
using var dataEntryStream = dataEntry.Open();
compressedStream.CopyTo(dataEntryStream);
compressedStream.Dispose();
}
}
if (options.DataFiles != null)
{
WriteFilesToArchive("data", options.DataFiles);
}
if (options.PluginFiles != null)
{
WriteFilesToArchive("plugins", options.PluginFiles);
}
}
ms.Position = 0;
return(ms);
}
public void FinalWrite()
{
CompressionHandler.FinalWrite(Stream);
}
public void Write(byte[] bytes, int offset, int length)
{
CompressionHandler.Write(bytes, offset, length, Stream);
}
